Compare commits

...
130 Commits
Author SHA1 Message Date
chrislu 27b34f3793 3.59 2023-11-12 22:12:08 -08:00
jsh 47112917ff ec.decode: mount the collected ec shards 2023-11-10 00:04:42 -08:00
chrislu 85bed4d92e Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-11-09 08:02:57 -08:00
chrislu c563386675 rename parameters 2023-11-09 08:02:39 -08:00
vados a1c6f1fbd6 chore(tests): update S3 compat tests
While present, the S3 compatibility tests were broken (at the very
least when running on Linux) due to a few issues:

- `ubuntu:latest` moving ahead of Python 3.6
- Docker networking not working as expected (host.docker.internal)

This commit fixes the s3 compatibility tests, ensuring they run
properly on linux, and updates the repository to contain some results.

Signed-off-by: vados <vados@vadosware.io>
2023-11-08 23:16:40 -08:00
Lars Lehtonen 5db82e594a weed/server: fix dropped webdav error 2023-11-07 19:05:21 -08:00
zehweh 2b9dda7d2e fix isMisplaced() in command_volume_fix_replication.go (#4988) 2023-11-07 07:58:19 -08:00
Konstantin Lebedev d693beea87 Replace an invalid method in a metric with an invalid one (#4987) 2023-11-06 10:51:47 -08:00
dependabot[bot] 8a6a7c63c0 Bump helm/chart-testing-action from 2.4.0 to 2.6.1 (#4986)
Bumps [helm/chart-testing-action](https://github.com/helm/chart-testing-action) from 2.4.0 to 2.6.1.
- [Release notes](https://github.com/helm/chart-testing-action/releases)
- [Commits](https://github.com/helm/chart-testing-action/compare/v2.4.0...v2.6.1)

---
updated-dependencies:
- dependency-name: helm/chart-testing-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:50:12 -08:00
dependabot[bot] 32cf06d19f Bump actions/dependency-review-action from 3.1.0 to 3.1.1 (#4985)
Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 3.1.0 to 3.1.1.
- [Release notes](https://github.com/actions/dependency-review-action/releases)
- [Commits](https://github.com/actions/dependency-review-action/compare/6c5ccdad469c9f8a2996bfecaec55a631a347034...9f45b2463b475767b61721ccfef113fef513e6aa)

---
updated-dependencies:
- dependency-name: actions/dependency-review-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:50:00 -08:00
dependabot[bot] 650f56e30c Bump github.com/aws/aws-sdk-go-v2/credentials from 1.13.43 to 1.15.1 (#4984)
Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.43 to 1.15.1.
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.15.1/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.13.43...config/v1.15.1)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/credentials
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:49:48 -08:00
dependabot[bot] 4d9274ec9d Bump google.golang.org/api from 0.148.0 to 0.149.0 (#4983)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.148.0 to 0.149.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.148.0...v0.149.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:49:34 -08:00
dependabot[bot] b3bcb1823e Bump github.com/arangodb/go-driver from 1.6.0 to 1.6.1 (#4982)
Bumps [github.com/arangodb/go-driver](https://github.com/arangodb/go-driver) from 1.6.0 to 1.6.1.
- [Release notes](https://github.com/arangodb/go-driver/releases)
- [Changelog](https://github.com/arangodb/go-driver/blob/master/CHANGELOG.md)
- [Commits](https://github.com/arangodb/go-driver/compare/v1.6.0...v1.6.1)

---
updated-dependencies:
- dependency-name: github.com/arangodb/go-driver
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:49:23 -08:00
dependabot[bot] 9f0e80e9f9 Bump github.com/rclone/rclone from 1.64.0 to 1.64.2 (#4981)
Bumps [github.com/rclone/rclone](https://github.com/rclone/rclone) from 1.64.0 to 1.64.2.
- [Release notes](https://github.com/rclone/rclone/releases)
- [Changelog](https://github.com/rclone/rclone/blob/master/RELEASE.md)
- [Commits](https://github.com/rclone/rclone/compare/v1.64.0...v1.64.2)

---
updated-dependencies:
- dependency-name: github.com/rclone/rclone
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-11-06 10:49:12 -08:00
chrislu a667f0ad99 fix error reporting
related to https://github.com/seaweedfs/seaweedfs/issues/4978
2023-11-05 07:17:16 -08:00
chrislu b40ba7c7fe Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-11-01 16:28:05 -07:00
chrislu 310eec9a9c skip if path changed
mitigate https://github.com/seaweedfs/seaweedfs/issues/4968
2023-11-01 16:27:59 -07:00
Nico D'Cotta 3834ad5c48 refactor webdav subdirectory, fixes #4967 (#4969)
* refactor webdav subdirectory, fixes #4967

* fix bug where Name() was not called on delegate wrappedFileInfo
2023-11-01 13:12:49 -07:00
tired-engineer 4ed06e9ba5 Pass correct Filer's gRPC port to S3 server. (#4972)
Fixes seaweedfs/seaweedfs#4971
2023-11-01 10:11:41 -07:00
Łukasz Maksymczuk d401e374f7 Add podLabels & podAnnotations to all pods (#4959)
The pods of all statefulsets/deployments can now have customizable
labels and annotations
2023-10-31 23:42:55 -07:00
dependabot[bot] 4bb3911fde Bump github.com/prometheus/procfs from 0.11.1 to 0.12.0 (#4966)
Bumps [github.com/prometheus/procfs](https://github.com/prometheus/procfs) from 0.11.1 to 0.12.0.
- [Release notes](https://github.com/prometheus/procfs/releases)
- [Commits](https://github.com/prometheus/procfs/compare/v0.11.1...v0.12.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/procfs
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 17:27:25 -07:00
dependabot[bot] 8cf83d3b12 Bump modernc.org/sqlite from 1.25.0 to 1.26.0 (#4965)
Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.25.0 to 1.26.0.
- [Commits](https://gitlab.com/cznic/sqlite/compare/v1.25.0...v1.26.0)

---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 17:27:17 -07:00
dependabot[bot] d440135d70 Bump github.com/rdleal/intervalst from 1.1.0 to 1.2.0 (#4964)
Bumps [github.com/rdleal/intervalst](https://github.com/rdleal/intervalst) from 1.1.0 to 1.2.0.
- [Commits](https://github.com/rdleal/intervalst/compare/v1.1.0...v1.2.0)

---
updated-dependencies:
- dependency-name: github.com/rdleal/intervalst
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 17:27:10 -07:00
dependabot[bot] 312abfbe82 Bump github.com/aws/aws-sdk-go-v2/credentials from 1.13.40 to 1.13.43 (#4963)
Bumps [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) from 1.13.40 to 1.13.43.
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.13.40...credentials/v1.13.43)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/credentials
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 17:27:02 -07:00
dependabot[bot] 28525fb500 Bump github.com/google/uuid from 1.3.1 to 1.4.0 (#4962)
Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.3.1 to 1.4.0.
- [Release notes](https://github.com/google/uuid/releases)
- [Changelog](https://github.com/google/uuid/blob/master/CHANGELOG.md)
- [Commits](https://github.com/google/uuid/compare/v1.3.1...v1.4.0)

---
updated-dependencies:
- dependency-name: github.com/google/uuid
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-30 17:26:54 -07:00
SmsS4 ba11331cc1 Fix twice increase metric counter (#4961) 2023-10-30 08:05:27 -07:00
SmsS4 f95848ba7d Add ErrorGetNotFound and ErrorGetInternal to volume server metrics (#4960) 2023-10-30 07:38:03 -07:00
chrislu d1e83a3b4d 3.58 2023-10-29 23:04:20 -07:00
duanhongyi bf839651ec Fix: issues #4950 (#4952) 2023-10-25 22:09:39 -07:00
zehweh 8b39bbbe2f fix copying .vif file in VolumeCopy (#4943)
closes #4934
fixes #2633
might fix #3528
2023-10-24 09:54:24 -07:00
fibbanachi 8dfb66880e Remove hardcoded podManagmentPolicy value in a helm chart (#4941) 2023-10-24 07:34:55 -07:00
dependabot[bot] 179039afaa Bump github.com/rdleal/intervalst from 0.0.0-20221028215511-a098aa0d2cb8 to 1.1.0 (#4940)
Bump github.com/rdleal/intervalst

Bumps [github.com/rdleal/intervalst](https://github.com/rdleal/intervalst) from 0.0.0-20221028215511-a098aa0d2cb8 to 1.1.0.
- [Commits](https://github.com/rdleal/intervalst/commits/v1.1.0)

---
updated-dependencies:
- dependency-name: github.com/rdleal/intervalst
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:10:17 -07:00
dependabot[bot] 53c709a580 Bump github.com/puzpuzpuz/xsync/v2 from 2.5.0 to 2.5.1 (#4939)
Bumps [github.com/puzpuzpuz/xsync/v2](https://github.com/puzpuzpuz/xsync) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/puzpuzpuz/xsync/releases)
- [Commits](https://github.com/puzpuzpuz/xsync/compare/v2.5.0...v2.5.1)

---
updated-dependencies:
- dependency-name: github.com/puzpuzpuz/xsync/v2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:09:39 -07:00
dependabot[bot] 85f41657dd Bump github.com/go-redsync/redsync/v4 from 4.9.4 to 4.10.0 (#4938)
Bumps [github.com/go-redsync/redsync/v4](https://github.com/go-redsync/redsync) from 4.9.4 to 4.10.0.
- [Release notes](https://github.com/go-redsync/redsync/releases)
- [Commits](https://github.com/go-redsync/redsync/compare/v4.9.4...v4.10.0)

---
updated-dependencies:
- dependency-name: github.com/go-redsync/redsync/v4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:09:26 -07:00
dependabot[bot] c999851555 Bump google.golang.org/api from 0.146.0 to 0.148.0 (#4937)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.146.0 to 0.148.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.146.0...v0.148.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:09:15 -07:00
dependabot[bot] aaf212f7ec Bump modernc.org/strutil from 1.1.3 to 1.2.0 (#4936)
Bumps [modernc.org/strutil](https://gitlab.com/cznic/strutil) from 1.1.3 to 1.2.0.
- [Commits](https://gitlab.com/cznic/strutil/compare/v1.1.3...v1.2.0)

---
updated-dependencies:
- dependency-name: modernc.org/strutil
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:09:05 -07:00
dependabot[bot] c6e9fd26dc Bump actions/checkout from 4.1.0 to 4.1.1 (#4935)
Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/8ade135a41bc03ea155e62e844d188df1ea18608...b4ffde65f46336ab88eb53be808477a3936bae11)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-23 13:08:43 -07:00
chrislu 6ae22b522b Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-10-22 12:59:36 -07:00
chrislu 2bc05f70e7 log full percentage 2023-10-22 12:59:34 -07:00
binbinshi 30b48ad614 see_dat add print name (#4930) 2023-10-20 06:13:54 -07:00
dependabot[bot] 294337d882 Bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.53.1 to 3.53.3 (#4928) 2023-10-19 11:45:16 -07:00
chrislu 612fd7206e fix 2023-10-19 08:10:33 -07:00
6c82d7ddfc [s3] do reload s3 static config (#4923)
* do reload s3 config

* print error on reload s3 config

* print success msg

* Update weed/s3api/s3api_server.go

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2023-10-18 23:26:49 -07:00
lakeland1990 3df0e11ad3 remove compare to fix #4668 issue (#4924) 2023-10-18 09:06:56 -07:00
dependabot[bot] 117fbba5bf Bump golang.org/x/tools from 0.13.0 to 0.14.0 (#4919)
Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.13.0 to 0.14.0.
- [Release notes](https://github.com/golang/tools/releases)
- [Commits](https://github.com/golang/tools/compare/v0.13.0...v0.14.0)

---
updated-dependencies:
- dependency-name: golang.org/x/tools
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 09:32:57 -07:00
dependabot[bot] b2ecfad35e Bump github.com/aws/aws-sdk-go-v2/service/s3 from 1.38.5 to 1.40.2 (#4918)
Bumps [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) from 1.38.5 to 1.40.2.
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.38.5...service/s3/v1.40.2)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/service/s3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 09:32:46 -07:00
dependabot[bot] 6acec5ed1c Bump github.com/aws/aws-sdk-go from 1.45.24 to 1.45.25 (#4917)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.45.24 to 1.45.25.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.45.24...v1.45.25)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 09:32:36 -07:00
dependabot[bot] 1a4c35d984 Bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.48.8 to 3.53.1 (#4916)
Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.48.8 to 3.53.1.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.48.8...v3.53.1)

---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 09:32:27 -07:00
dependabot[bot] 3469ad6b3f Bump github.com/shirou/gopsutil/v3 from 3.23.6 to 3.23.9 (#4915)
Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.23.6 to 3.23.9.
- [Release notes](https://github.com/shirou/gopsutil/releases)
- [Commits](https://github.com/shirou/gopsutil/compare/v3.23.6...v3.23.9)

---
updated-dependencies:
- dependency-name: github.com/shirou/gopsutil/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-16 09:32:19 -07:00
Konstantin LebedevandKonstantin Lebedev 6175102ccf add make benchmark with WARP (#4914)
add make benchmark

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
2023-10-15 23:02:27 -07:00
Yuval Yacoby c6991dfd5f Helm use external certificates (#4913) 2023-10-15 09:14:24 -07:00
chrislu cbc24c7b24 disconnect from old subscribers 2023-10-13 16:48:14 -07:00
James Hedley 6acb43bbbb Add optional flags to enable mTLS with verification of client certificate (#4910) 2023-10-13 09:02:24 -07:00
Konstantin Lebedev ce9b822a8e golang up version to 1.21.3 (#4911) 2023-10-13 05:31:43 -07:00
chrislu b62066d38f adding DiskType to /dir/status output 2023-10-12 21:39:11 -07:00
chrislu deb86ff4a6 upgrading grpc 2023-10-12 21:38:34 -07:00
chrislu de0b969b36 Revert "rename"
This reverts commit 35b5264ab7.
2023-10-12 20:28:11 -07:00
Konstantin LebedevandKonstantin Lebedev 1cac5d983d fix: disallow file name too long when writing a file (#4881)
* fix: disallow file name too long when writing a file

* bool LongerName to MaxFilenameLength

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
2023-10-12 14:29:55 -07:00
Konstantin LebedevandKonstantin Lebedev edee91ef0e fix has changes false if deleted result size eq zero (#4909)
* fix has changes false if deleted result size eq zero
https://github.com/seaweedfs/seaweedfs/issues/4171

* fix change respos status 202 to 304 if the file has already been deleted
https://github.com/seaweedfs/seaweedfs/issues/4171

* fix delete if set apply changes
https://github.com/seaweedfs/seaweedfs/issues/4171

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
2023-10-12 08:15:20 -07:00
chrislu 35b5264ab7 rename 2023-10-11 21:44:56 -07:00
dependabot[bot] fb450b2a5d Bump golang.org/x/net from 0.15.0 to 0.17.0 (#4908) 2023-10-11 17:40:13 -07:00
Amir H. Yeganemehr 3e8604a4c5 add fs.mergeVolumes command into shell (#4907)
* add fs.compact command into shell

* rename fs.compact to fs.mergeVolumes
2023-10-10 12:47:18 -07:00
dependabot[bot] 08a45d6c84 Bump github.com/spf13/viper from 1.16.0 to 1.17.0 (#4906)
Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.16.0 to 1.17.0.
- [Release notes](https://github.com/spf13/viper/releases)
- [Commits](https://github.com/spf13/viper/compare/v1.16.0...v1.17.0)

---
updated-dependencies:
- dependency-name: github.com/spf13/viper
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 10:01:51 -07:00
dependabot[bot] c2d338160e Bump github.com/aws/aws-sdk-go from 1.45.19 to 1.45.24 (#4904)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.45.19 to 1.45.24.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.45.19...v1.45.24)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 09:59:56 -07:00
dependabot[bot] 9662d71a3a Bump github.com/rabbitmq/amqp091-go from 1.8.1 to 1.9.0 (#4905)
Bumps [github.com/rabbitmq/amqp091-go](https://github.com/rabbitmq/amqp091-go) from 1.8.1 to 1.9.0.
- [Release notes](https://github.com/rabbitmq/amqp091-go/releases)
- [Changelog](https://github.com/rabbitmq/amqp091-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rabbitmq/amqp091-go/compare/v1.8.1...v1.9.0)

---
updated-dependencies:
- dependency-name: github.com/rabbitmq/amqp091-go
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 09:59:44 -07:00
dependabot[bot] 9242a6cc5d Bump google.golang.org/api from 0.141.0 to 0.146.0 (#4903)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.141.0 to 0.146.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.141.0...v0.146.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 09:59:24 -07:00
dependabot[bot] 49a0ee60b3 Bump github.com/tidwall/gjson from 1.16.0 to 1.17.0 (#4902)
Bumps [github.com/tidwall/gjson](https://github.com/tidwall/gjson) from 1.16.0 to 1.17.0.
- [Commits](https://github.com/tidwall/gjson/compare/v1.16.0...v1.17.0)

---
updated-dependencies:
- dependency-name: github.com/tidwall/gjson
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-10-09 09:59:15 -07:00
Konstantin LebedevandKonstantin Lebedev 2b3e39397e fix: skipping checking active volumes with the same number of files at the moment (#4893)
* fix: skipping checking active volumes with the same number of files at the moment
 https://github.com/seaweedfs/seaweedfs/issues/4140

* refactor with comments
https://github.com/seaweedfs/seaweedfs/issues/4140

* add TestShouldSkipVolume

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
2023-10-09 09:57:26 -07:00
Yuval Yacoby 3fe00996b2 added healthz endpoint to filer (#4899) 2023-10-08 07:03:18 -07:00
Andrew Garrett 082f67bfe9 Fix filer.backup local sink to propagate file mode changes (#4896) 2023-10-06 05:40:20 -07:00
chrislu 0fd7222d65 default to skip if less than 4 nodes 2023-10-05 11:13:48 -07:00
chrislu 7b3c63a8af handle wrapped errors 2023-10-05 08:15:11 -07:00
chrislu 4e493d596c minor 2023-10-05 08:13:53 -07:00
Yuval YacobyandYuval Yacoby 659133fb68 Helm chart jwt signing configuration (#4894)
* add helm chart config to customize jwt signing

* restore values

---------

Co-authored-by: Yuval Yacoby <yyacoby@paloaltonetworks.com>
2023-10-05 07:46:28 -07:00
Lars Lehtonen 530bdedf76 weed/operation: fix dropped error (#4888) 2023-10-03 10:55:20 -07:00
chrislu 6ebe26a765 Revert "Revert "Revert "Add disk type to prometheus metrics" (#4777)""
This reverts commit 567d788928.
2023-10-03 08:28:52 -07:00
chrislu 7540d43ee9 Revert "Revert "fix compilation""
This reverts commit f9abfd0b03.
2023-10-03 08:28:02 -07:00
chrislu 249c0e06ef Revert "fix compilation"
This reverts commit 451ec6504d.
2023-10-03 08:27:50 -07:00
chrislu a78c4141e5 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-10-03 08:15:20 -07:00
chrislu 451ec6504d fix compilation 2023-10-03 08:15:18 -07:00
chrislu f9abfd0b03 Revert "fix compilation"
This reverts commit 0483ba3889.
2023-10-03 07:58:11 -07:00
fibbanachi 763d1946f9 add ingress for seaweedfs s3 (#4887) 2023-10-03 07:35:52 -07:00
chrislu 0483ba3889 fix compilation 2023-10-02 11:55:50 -07:00
chrislu 567d788928 Revert "Revert "Add disk type to prometheus metrics" (#4777)"
This reverts commit 9215ba24be.
2023-10-02 11:49:54 -07:00
chrislu b5070d6280 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-10-02 10:52:40 -07:00
dependabot[bot] 5fb90a3157 Bump github.com/aws/aws-sdk-go from 1.44.314 to 1.45.19 (#4886) 2023-10-02 08:46:53 -07:00
dependabot[bot] a16c0fabca Bump github.com/prometheus/client_golang from 1.16.0 to 1.17.0 (#4884) 2023-10-02 08:46:40 -07:00
dependabot[bot] 58b46229aa Bump gocloud.dev/pubsub/natspubsub from 0.33.0 to 0.34.0 (#4883) 2023-10-02 08:46:25 -07:00
dependabot[bot] c49dee4dad Bump cloud.google.com/go/storage from 1.32.0 to 1.33.0 (#4882) 2023-10-02 08:46:07 -07:00
dependabot[bot] f7ad82e03d Bump golang.org/x/image from 0.11.0 to 0.12.0 (#4885) 2023-10-02 08:45:33 -07:00
Konstantin Lebedev dffe00a822 fix: logger place msg (#4880) 2023-10-02 08:29:09 -07:00
chrislu bf13f3ced7 add subscriber coordinator 2023-10-02 01:02:27 -07:00
chrislu 734178093e refactor TopicPartition struct 2023-10-02 01:01:45 -07:00
chrislu 2a578b9033 refactor 2023-10-01 11:59:19 -07:00
chrislu 321fda552c comment 2023-10-01 11:36:27 -07:00
chrislu b02fdeabff RetryForever => RetryUntil 2023-10-01 11:33:56 -07:00
chrislu ebd6f96d35 adjust mq.proto 2023-09-30 21:47:57 -07:00
chrislu 02cb799481 add a test framework for grpc 2023-09-30 21:36:58 -07:00
chrislu c4e6545197 fix compilation 2023-09-30 21:26:09 -07:00
chrislu 358cba43ef update proto generated files 2023-09-30 13:19:25 -07:00
chrislu 597ab1e4bf add design for mq balancer 2023-09-30 13:19:05 -07:00
chrislu d7c211fe8c adjust mq proto 2023-09-30 13:18:49 -07:00
chrislu 40888b5612 align 64bit atomic operation
fix https://github.com/seaweedfs/seaweedfs/issues/4878
2023-09-28 07:58:43 -07:00
aronneagu abe599ee4a Allow override internalTrafficPolicy (#4877) 2023-09-28 06:18:54 -07:00
dd580190b4 fix: avoid deleting one replica without sync (#4875)
* fix: avoid deleting one replica without sync
https://github.com/seaweedfs/seaweedfs/issues/4647

* Update weed/shell/command_volume_fix_replication.go

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>

* fix: revert this existing do option to positive

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2023-09-27 23:12:10 -07:00
Konstantin Lebedev 44906f1f3b fix: avoid error file name too long when writing a file (#4876) 2023-09-27 05:40:51 -07:00
chrislu 9d589b48e6 rename function 2023-09-26 15:17:33 -07:00
chrislu 17a0f3b0e0 adjust parameter names 2023-09-26 15:17:12 -07:00
Konstantin LebedevandKonstantin Lebedev df4ded758e fix: avoid deleting more than one replica (#4873)
https://github.com/seaweedfs/seaweedfs/issues/4647

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
2023-09-26 00:20:48 -07:00
chrislu 19505c1cf4 describe a topic 2023-09-25 20:46:00 -07:00
Tobias Gurtzick 78dbac7702 fix(ec): volumes created by foreign collection due to bug in ec balance (#4864)
* fix(ec): ignore 0 byte data files

refers to parts of #4861

Signed-off-by: Tobias Gurtzick <magic@wizardtales.com>

* fix(ec): ignore volumes not from the current collection during balance

fixes #4861

Signed-off-by: Tobias Gurtzick <magic@wizardtales.com>

---------

Signed-off-by: Tobias Gurtzick <magic@wizardtales.com>
2023-09-25 10:35:43 -07:00
chrislu faabc695b6 update go mod 2023-09-25 09:41:02 -07:00
chrislu 0530e90ea1 Update go.mod 2023-09-25 09:37:45 -07:00
chrislu 645ae8c57b Revert "Revert "Merge branch 'master' of https://github.com/seaweedfs/seaweedfs""
This reverts commit 8cb42c39
2023-09-25 09:35:16 -07:00
chrislu 3d07895518 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-09-25 09:16:35 -07:00
chrislu a6e35e4f68 fix compilation 2023-09-25 09:16:33 -07:00
dependabot[bot] a132d44556 Bump google.golang.org/grpc from 1.58.0 to 1.58.2 (#4872)
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.58.0 to 1.58.2.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.58.0...v1.58.2)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 09:15:21 -07:00
dependabot[bot] a345b6802d Bump github.com/go-redsync/redsync/v4 from 4.8.2 to 4.9.4 (#4871)
Bumps [github.com/go-redsync/redsync/v4](https://github.com/go-redsync/redsync) from 4.8.2 to 4.9.4.
- [Release notes](https://github.com/go-redsync/redsync/releases)
- [Commits](https://github.com/go-redsync/redsync/compare/v4.8.2...v4.9.4)

---
updated-dependencies:
- dependency-name: github.com/go-redsync/redsync/v4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 09:15:07 -07:00
dependabot[bot] 51efba43fd Bump gocloud.dev/pubsub/rabbitpubsub from 0.33.0 to 0.34.0 (#4870)
Bumps [gocloud.dev/pubsub/rabbitpubsub](https://github.com/google/go-cloud) from 0.33.0 to 0.34.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.33.0...v0.34.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/rabbitpubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 09:14:49 -07:00
dependabot[bot] adabb63c18 Bump github.com/aws/aws-sdk-go-v2/config from 1.18.39 to 1.18.42 (#4868)
Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.18.39 to 1.18.42.
- [Release notes](https://github.com/aws/aws-sdk-go-v2/releases)
- [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.39...config/v1.18.42)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go-v2/config
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 09:14:29 -07:00
f8b94cac0e [s3acl] Step1: move s3account.AccountManager into to iam.S3ApiConfiguration (#4859)
* move s3account.AccountManager into to iam.S3ApiConfiguration and switch to Interface

https://github.com/seaweedfs/seaweedfs/issues/4519

* fix: test bucket acl default and
adjust the variable names

* fix: s3 api config test

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2023-09-25 08:34:12 -07:00
chrislu c9177c92e5 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-09-25 08:20:16 -07:00
chrislu 81e57bd45e fix test 2023-09-25 08:20:03 -07:00
dependabot[bot] 7bb58e23ea Bump actions/checkout from 4.0.0 to 4.1.0 (#4867)
Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/3df4ab11eba7bda6032a0b82a6bb43b11571feac...8ade135a41bc03ea155e62e844d188df1ea18608)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-09-25 08:18:11 -07:00
Lars Lehtonen 28a3a31b27 weed/replication/sub: fix dropped error (#4865) 2023-09-25 07:33:18 -07:00
Konstantin Lebedev a40888d343 fix: make dev_tls with volume https (#4866) 2023-09-25 06:47:38 -07:00
chrislu 23f334dd13 comment 2023-09-24 23:36:43 -07:00
chrislu c7e05e4e71 ensure latest stats are reported 2023-09-24 23:34:31 -07:00
chrislu dff2ce5d2f read existing broker stats 2023-09-24 23:10:33 -07:00
chrislu 1492bf7552 fix listing topics 2023-09-24 23:05:41 -07:00
191 changed files with 5083 additions and 2762 deletions

No files matched your search

+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -87,7 +87,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
with:
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
-
name: Docker meta
id: docker_meta
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: 'Dependency Review'
uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034
uses: actions/dependency-review-action@9f45b2463b475767b61721ccfef113fef513e6aa
+1 -1
View File
@@ -30,7 +30,7 @@ jobs:
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Install dependencies
run: |
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Get dependencies
run: |
+1 -1
View File
@@ -12,7 +12,7 @@ jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@master
with:
+2 -2
View File
@@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
with:
fetch-depth: 0
@@ -31,7 +31,7 @@ jobs:
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.4.0
uses: helm/chart-testing-action@v2.6.1
- name: Run chart-testing (list-changed)
id: list-changed
+23 -2
View File
@@ -1,14 +1,35 @@
BINARY = weed
SOURCE_DIR = .
debug ?= 0
all: install
install:
cd weed; go install
warp_install:
go install github.com/minio/warp@v0.7.6
full_install:
cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
tests:
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
server: install
weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json
benchmark: install warp_install
pkill weed || true
pkill warp || true
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
warp client &
while ! nc -z localhost 8000 ; do sleep 1 ; done
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
pkill warp
pkill weed
# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
benchmark_with_pprof: debug = 1
benchmark_with_pprof: benchmark
test:
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+65
View File
@@ -582,6 +582,71 @@ Percentage of the requests served within a certain time (ms)
100% 54.1 ms
```
### Run WARP and launch a mixed benchmark. ###
```
make benchmark
warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"
Mixed operations.
Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
* Throughput: 6.19 obj/s
Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
* Throughput: 279.85 MiB/s, 27.99 obj/s
Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
* Throughput: 89.86 MiB/s, 8.99 obj/s
Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
* Throughput: 18.63 obj/s
Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
```
To see segmented request statistics, use the --analyze.v parameter.
```
warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
18642 operations loaded... Done!
Mixed operations.
----------------------------------------
Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
* Throughput: 6.19 obj/s
Requests considered: 1855:
* Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
----------------------------------------
Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
* Throughput: 279.77 MiB/s, 27.98 obj/s
Requests considered: 8389:
* Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
* TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
* First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
* First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
* Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
* Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
----------------------------------------
Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
* Throughput: 89.83 MiB/s, 8.98 obj/s
Requests considered: 2689:
* Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
----------------------------------------
Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
* Throughput: 18.63 obj/s
Requests considered: 5587:
* Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
* First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
* Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
Total Errors:0.
```
[Back to TOC](#table-of-contents)
## License ##
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.20-alpine as builder
FROM golang:1.21-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.20-buster as builder
FROM golang:1.21-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.20-buster as builder
FROM golang:1.21-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+6 -6
View File
@@ -102,12 +102,12 @@ clean:
rm ./weed
certstrap:
go get github.com/square/certstrap
certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
go install -v github.com/square/certstrap@latest
certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
+12 -2
View File
@@ -40,7 +40,10 @@
"List",
"Tagging",
"Write"
]
],
"account": {
"id": "testid"
}
},
{
"name": "s3_tests_alt",
@@ -101,5 +104,12 @@
"Write"
]
}
]
],
"accounts": [
{
"id" : "testid",
"displayName": "M. Tester",
"emailAddress": "tester@ceph.com"
}
]
}
+2 -2
View File
@@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}-
[s3 main]
# main display_name set in vstart.sh
display_name = s3_tests
display_name = M. Tester
# main user_idname set in vstart.sh
user_id = s3_tests
user_id = testid
# main email set in vstart.sh
email = tester@ceph.com
+4
View File
@@ -12,5 +12,9 @@ WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,clie
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_HTTPS_CLIENT_ENABLE=true
WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
#GRPC_GO_LOG_SEVERITY_LEVEL=info
#GRPC_GO_LOG_VERBOSITY_LEVEL=2
+107 -86
View File
@@ -1,22 +1,22 @@
module github.com/seaweedfs/seaweedfs
go 1.20
go 1.21
require (
cloud.google.com/go v0.110.7 // indirect
cloud.google.com/go v0.110.8 // indirect
cloud.google.com/go/pubsub v1.33.0
cloud.google.com/go/storage v1.32.0
cloud.google.com/go/storage v1.33.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.44.314
github.com/aws/aws-sdk-go v1.45.25
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cenkalti/backoff/v4 v4.2.1
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.1
@@ -32,7 +32,7 @@ require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.8.2
github.com/go-redsync/redsync/v4 v4.10.0
github.com/go-sql-driver/mysql v1.7.1
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/gocql/gocql v1.6.0
@@ -41,8 +41,7 @@ require (
github.com/golang/protobuf v1.5.3
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.1
github.com/google/uuid v1.4.0
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gorilla/mux v1.8.0
@@ -58,7 +57,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/reedsolomon v1.11.8
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.9
@@ -66,7 +65,7 @@ require (
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.11 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -74,26 +73,25 @@ require (
github.com/olivere/elastic/v7 v7.0.32
github.com/peterh/liner v1.2.2
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.2.0
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.11.1
github.com/prometheus/client_golang v1.17.0
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.12.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.3
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.16.0
github.com/spf13/viper v1.17.0
github.com/stretchr/testify v1.8.4
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/tidwall/gjson v1.16.0
github.com/tidwall/gjson v1.17.0
github.com/tidwall/match v1.1.1
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -110,21 +108,21 @@ require (
go.mongodb.org/mongo-driver v1.12.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.34.0
gocloud.dev/pubsub/natspubsub v0.33.0
gocloud.dev/pubsub/rabbitpubsub v0.33.0
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
golang.org/x/image v0.11.0
golang.org/x/net v0.15.0
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sys v0.12.0
gocloud.dev/pubsub/natspubsub v0.34.0
gocloud.dev/pubsub/rabbitpubsub v0.34.0
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/image v0.12.0
golang.org/x/net v0.17.0
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.13.0
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.11.0
golang.org/x/tools v0.14.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.141.0
google.golang.org/api v0.149.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/grpc v1.58.0
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
@@ -134,19 +132,19 @@ require (
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.6.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.25.0
modernc.org/strutil v1.1.3
modernc.org/sqlite v1.26.0
modernc.org/strutil v1.2.0
modernc.org/token v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.4.0
github.com/arangodb/go-driver v1.6.0
github.com/arangodb/go-driver v1.6.1
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/config v1.18.39
github.com/aws/aws-sdk-go-v2/credentials v1.13.37
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
github.com/aws/aws-sdk-go-v2 v1.22.1
github.com/aws/aws-sdk-go-v2/config v1.18.42
github.com/aws/aws-sdk-go-v2/credentials v1.15.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
github.com/fluent/fluent-logger-golang v1.9.0
github.com/golang-jwt/jwt/v5 v5.0.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -154,22 +152,22 @@ require (
github.com/hashicorp/raft v1.5.0
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/puzpuzpuz/xsync/v2 v2.5.0
github.com/rabbitmq/amqp091-go v1.8.1
github.com/rclone/rclone v1.63.1
github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8
github.com/puzpuzpuz/xsync/v2 v2.5.1
github.com/rabbitmq/amqp091-go v1.9.0
github.com/rclone/rclone v1.64.2
github.com/rdleal/intervalst v1.2.0
github.com/schollz/progressbar/v3 v3.13.1
github.com/shirou/gopsutil/v3 v3.23.5
github.com/shirou/gopsutil/v3 v3.23.9
github.com/tikv/client-go/v2 v2.0.7
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8
github.com/ydb-platform/ydb-go-sdk/v3 v3.53.3
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
require (
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute v1.23.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
cloud.google.com/go/iam v1.1.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
@@ -178,83 +176,103 @@ require (
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
github.com/ProtonMail/go-srp v0.0.7 // indirect
github.com/ProtonMail/gopenpgp/v2 v2.7.3 // indirect
github.com/PuerkitoBio/goquery v1.8.1 // indirect
github.com/Unknwon/goconfig v1.0.0 // indirect
github.com/abbot/go-http-auth v0.4.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 // indirect
github.com/aws/smithy-go v1.16.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.13.1 // indirect
github.com/buengese/sgzip v0.1.1 // indirect
github.com/calebcase/tmpfile v1.0.3 // indirect
github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/emersion/go-message v0.17.0 // indirect
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/fclairamb/go-log v0.4.1 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/go-chi/chi/v5 v5.0.8 // indirect
github.com/go-chi/chi/v5 v5.0.10 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-resty/resty/v2 v2.7.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/hashicorp/go-hclog v1.5.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/henrybear327/Proton-API-Bridge v0.0.0-20230908065933-5bfa15b567db // indirect
github.com/henrybear327/go-proton-api v0.0.0-20230907193451-e563407504ce // indirect
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jlaffaye/ftp v0.2.0 // indirect
github.com/jonboulle/clockwork v0.3.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/montanaflynn/stats v0.7.0 // indirect
github.com/nats-io/nats.go v1.28.0 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nats.go v1.30.2 // indirect
github.com/nats-io/nkeys v0.4.5 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
github.com/ncw/swift/v2 v2.0.1 // indirect
github.com/ncw/swift/v2 v2.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.45.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
@@ -267,28 +285,30 @@ require (
github.com/pkg/xattr v0.4.9 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 // indirect
github.com/relvacode/iso8601 v1.3.0 // indirect
github.com/rfjakob/eme v1.1.2 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
github.com/smartystreets/goconvey v1.8.1 // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd // indirect
github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -301,20 +321,21 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
storj.io/drpc v0.0.32 // indirect
storj.io/uplink v1.10.0 // indirect
storj.io/common v0.0.0-20230907123639-5fd0608fd947 // indirect
storj.io/drpc v0.0.33 // indirect
storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
storj.io/uplink v1.12.0 // indirect
)
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
+312 -236
View File
File diff suppressed because it is too large. Load diff
+2 -2
View File
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.57"
version: 3.57.0
appVersion: "3.59"
version: 3.59.0
+1 -1
View File
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Certificate
metadata:
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Issuer
metadata:
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: ClusterIssuer
metadata:
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Certificate
metadata:
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Certificate
metadata:
@@ -12,7 +12,7 @@ metadata:
app.kubernetes.io/component: filer
spec:
serviceName: {{ template "seaweedfs.name" . }}-filer
podManagementPolicy: Parallel
podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
replicas: {{ .Values.filer.replicas }}
{{- if (gt (int .Values.filer.updatePartition) 0) }}
updateStrategy:
@@ -33,6 +33,13 @@ spec:
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: filer
{{ with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{ with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
{{- if .Values.filer.affinity }}
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Certificate
metadata:
@@ -11,7 +11,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
serviceName: {{ template "seaweedfs.name" . }}-master
podManagementPolicy: Parallel
podManagementPolicy: {{ .Values.master.podManagementPolicy }}
replicas: {{ .Values.master.replicas }}
{{- if (gt (int .Values.master.updatePartition) 0) }}
updateStrategy:
@@ -32,6 +32,13 @@ spec:
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: master
{{ with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{ with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
{{- if .Values.master.affinity }}
@@ -24,6 +24,13 @@ spec:
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: s3
{{ with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{ with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
{{- if .Values.s3.tolerations }}
@@ -0,0 +1,46 @@
{{- if .Values.s3.ingress.enabled }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
apiVersion: networking.k8s.io/v1beta1
{{- else }}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: ingress-{{ template "seaweedfs.name" . }}-s3
namespace: {{ .Release.Namespace }}
{{- if .Values.s3.ingress.annotations }}
annotations:
{{ tpl .Values.s3.ingress.annotations . | nindent 4 | trim }}
{{- end }}
labels:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: s3
spec:
ingressClassName: {{ .Values.s3.ingress.className | quote }}
tls:
{{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
rules:
- http:
paths:
- path: /
pathType: ImplementationSpecific
backend:
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
service:
name: {{ template "seaweedfs.name" . }}-s3
port:
number: {{ .Values.s3.port }}
#name:
{{- else }}
serviceName: {{ template "seaweedfs.name" . }}-s3
servicePort: {{ .Values.s3.port }}
{{- end }}
{{- if .Values.s3.ingress.host }}
host: {{ .Values.s3.ingress.host }}
{{- end }}
{{- end }}
@@ -9,6 +9,7 @@ metadata:
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
ports:
- name: "swfs-s3"
port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
@@ -13,10 +13,38 @@ data:
security.toml: |-
# this file is read by master, volume server, and filer
{{- if .Values.global.securityConfig.jwtSigning.volumeWrite }}
# the jwt signing key is read by master and volume server
# a jwt expires in 10 seconds
[jwt.signing]
key = "{{ randAlphaNum 10 | b64enc }}"
{{- end }}
{{- if .Values.global.securityConfig.jwtSigning.volumeRead }}
# this jwt signing key is read by master and volume server, and it is used for read operations:
# - the Master server generates the JWT, which can be used to read a certain file on a volume server
# - the Volume server validates the JWT on reading
[jwt.signing.read]
key = "{{ randAlphaNum 10 | b64enc }}"
{{- end }}
{{- if .Values.global.securityConfig.jwtSigning.filerWrite }}
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
# - f.e. the S3 API Shim generates the JWT
# - the Filer server validates the JWT on writing
# the jwt defaults to expire after 10 seconds.
[jwt.filer_signing]
key = "{{ randAlphaNum 10 | b64enc }}"
{{- end }}
{{- if .Values.global.securityConfig.jwtSigning.filerRead }}
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
# - f.e. the S3 API Shim generates the JWT
# - the Filer server validates the JWT on writing
# the jwt defaults to expire after 10 seconds.
[jwt.filer_signing.read]
key = "{{ randAlphaNum 10 | b64enc }}"
{{- end }}
# all grpc tls authentications are mutual
# the values for the following ca, cert, and key are paths to the PERM files.
@@ -1,4 +1,4 @@
{{- if .Values.global.enableSecurity }}
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
kind: Certificate
metadata:
@@ -10,6 +10,7 @@ metadata:
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
clusterIP: None
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
ports:
- name: "swfs-volume"
port: {{ .Values.volume.port }}
@@ -27,4 +28,4 @@ spec:
{{- end }}
selector:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
app.kubernetes.io/component: volume
app.kubernetes.io/component: volume
@@ -12,7 +12,7 @@ metadata:
spec:
serviceName: {{ template "seaweedfs.name" . }}-volume
replicas: {{ .Values.volume.replicas }}
podManagementPolicy: Parallel
podManagementPolicy: {{ .Values.volume.podManagementPolicy }}
selector:
matchLabels:
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -26,6 +26,13 @@ spec:
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: volume
{{ with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{ with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- if .Values.volume.affinity }}
affinity:
+31
View File
@@ -10,6 +10,12 @@ global:
restartPolicy: Always
loggingLevel: 1
enableSecurity: false
securityConfig:
jwtSigning:
volumeWrite: true
volumeRead: false
filerWrite: false
filerRead: false
certificates:
alphacrds: false
monitoring:
@@ -88,6 +94,9 @@ master:
extraVolumes: ""
extraVolumeMounts: ""
## Set podManagementPolicy
podManagementPolicy: Parallel
# Resource requests, limits, etc. for the master cluster placement. This
# should map directly to the value of the resources field for a PodSpec,
# formatted as a multi-line string. By default no direct resource request
@@ -263,6 +272,9 @@ volume:
extraVolumes: ""
extraVolumeMounts: ""
## Set podManagementPolicy
podManagementPolicy: Parallel
# Affinity Settings
# Commenting out or setting as empty the affinity variable, will allow
# deployment to single node services such as Minikube
@@ -396,6 +408,9 @@ filer:
extraVolumes: ""
extraVolumeMounts: ""
## Set podManagementPolicy
podManagementPolicy: Parallel
# Affinity Settings
# Commenting out or setting as empty the affinity variable, will allow
# deployment to single node services such as Minikube
@@ -621,6 +636,12 @@ s3:
failureThreshold: 100
timeoutSeconds: 10
ingress:
enabled: false
className: "nginx"
# host: false for "*" hostname
host: "seaweedfs.cluster.local"
certificates:
commonName: "SeaweedFS CA"
ipAddresses: []
@@ -628,3 +649,13 @@ certificates:
keySize: 2048
duration: 2160h # 90d
renewBefore: 360h # 15d
externalCertificates:
# This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
# you will need to store your provided certificates in the secret read by the different services:
# seaweedfs-master-cert, seaweedfs-filer-cert, etc. Can see any statefulset definition to see secret names
enabled: false
# Labels to be added to all the created pods
podLabels: {}
# Annotations to be added to all the created pods
podAnnotations: {}
@@ -13,12 +13,12 @@ public class FilerClient extends FilerGrpcClient {
private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
public FilerClient(String host, int grpcPort) {
super(host, grpcPort-10000, grpcPort);
public FilerClient(String filerHost, int filerGrpcPort) {
super(filerHost, filerGrpcPort-10000, filerGrpcPort);
}
public FilerClient(String host, int port, int grpcPort) {
super(host, port, grpcPort);
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
super(filerHost, filerPort, filerGrpcPort);
}
public static String toFileId(FilerProto.FileId fid) {
@@ -424,6 +424,7 @@ message FilerConf {
string data_center = 9;
string rack = 10;
string data_node = 11;
uint32 max_file_name_length = 12;
}
repeated PathConf locations = 2;
}
+6
View File
@@ -1,2 +1,8 @@
# Test run scratch data
/s3-tests
/tmp
# Test run outputs
weed.log
compat.raw.txt
compat.summary.txt
+13 -3
View File
@@ -1,11 +1,21 @@
# the tests only support python 3.6, not newer
FROM ubuntu:latest
#FROM ubuntu:latest
FROM python:3.6.15-slim-buster
RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
# Installed required system deps
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
# Install python deps
RUN pip install virtualenv
# Clone Ceph S3 tests
RUN git clone https://github.com/ceph/s3-tests.git
WORKDIR s3-tests
# we pin a certain commit
# Pin to a certain commit on ceph/s3-tests
# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
RUN ./bootstrap
+9
View File
@@ -0,0 +1,9 @@
.PHONY: all build-image test
all: build-image test
build-image:
./prepare.sh
test:
./run.sh
+236 -4
View File
@@ -1,13 +1,245 @@
# Running S3 Compatibility tests against SeaweedFS
This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
This is using [the tests from CephFS][s3-tests].
[s3-tests]: https://github.com/ceph/s3-tests
## Prerequisites
- have Docker installed
- have [Docker][docker] installed
- this has been executed on Mac. On Linux, the hostname in `s3tests.conf` needs to be adjusted.
[docker]: https://docs.docker.com
## Running tests
- `./prepare.sh` to build the docker image
- `./run.sh` to execute all tests
To build the docker image that is used for the tests:
```console
./prepare.sh
```
To execute all tests:
```console
./run.sh
```
To see debug output including all commands run by the script:
```console
DEBUG=y ./run.sh
```
> [!WARNING]
>
> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
> the container that is runnin the S3 compatibility tests.
>
> There are at least a couple ways to solve this:
>
> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
>
> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
> visit unexpected websites or use the host-passthrough internet access maliciously.
>
> If you are OK with the risk of allowing `--net=host`:
>
> - Set `host = localhost` in `s3tests.conf`
> - Set `DOCKER_NET_HOST=y` when running `run.sh`
## Most recent results
See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
The file is reproduced below for ease of access:
```
/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
warnings.warn(warning, PythonDeprecationWarning)
s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
s3tests_boto3.functional.test_s3.test_object_write_file ... ok
s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
ERROR
======================================================================
ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
----------------------------------------------------------------------
Traceback (most recent call last):
File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
self.test(*self.arg)
File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
'DisplayName': acl_response['Owner']['DisplayName'],
KeyError: 'Owner'
-------------------- >> begin captured logging << --------------------
```
+186
View File
@@ -0,0 +1,186 @@
/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
warnings.warn(warning, PythonDeprecationWarning)
s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
s3tests_boto3.functional.test_s3.test_object_write_file ... ok
s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
ERROR
======================================================================
ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
----------------------------------------------------------------------
Traceback (most recent call last):
File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
self.test(*self.arg)
File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
'DisplayName': acl_response['Owner']['DisplayName'],
KeyError: 'Owner'
-------------------- >> begin captured logging << --------------------
+75 -11
View File
@@ -1,24 +1,88 @@
#!/usr/bin/env bash
set -ex
CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
CONF_FILE=${CONF_FILE:-s3tests.conf}
WEED_BIN=${WEED_BIN:-../../../weed/weed}
TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
# Set up debugging for this bash script if DEBUG is set
if [ -n "${DEBUG}" ]; then
echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
set -ex
fi
# Reset from possible previous test run
killall -9 weed || echo "already stopped"
rm -Rf tmp
mkdir tmp
docker stop s3test-instance || echo "already stopped"
docker stop $CONTAINER_NAME || echo "already stopped"
# Ensure ulimit is set to reasonable value
ulimit -n 10000
../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
# Start weed w/ filer + s3 in the background
$WEED_BIN server \
-filer \
-s3 \
-volume.max 0 \
-master.volumeSizeLimitMB 5 \
-dir "$(pwd)/tmp" \
1>&2>weed.log &
# Wait for master to start up
echo -e "\n[info] waiting for master @ 9333...";
until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
printf '.'
sleep 5
printf '.';
sleep 5;
done
sleep 3
sleep 3;
rm -Rf logs-full.txt logs-summary.txt
# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v -a 'resource=object,!bucket-policy,!versioning,!encryption'
docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
# Wait for s3 to start up
echo -e "\n[info] waiting for S3 @ 8333...";
until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
printf '.';
sleep 5;
done
sleep 3;
docker stop s3test-instance || echo "already stopped"
killall -9 weed
# Determine whether docker net
DOCKER_NET_HOST_ARGS=""
if [ -n "${DOCKER_NET_HOST}" ]; then
DOCKER_NET_HOST_ARGS="--net=host"
echo -e "\n[info] setting docker to het nost"
fi
echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
echo -e "\n[info] running tests with unfiltered output...";
docker run \
--name $CONTAINER_NAME \
--rm \
${DOCKER_NET_HOST_ARGS} \
-e S3TEST_CONF=$CONF_FILE \
-v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
-it \
s3tests \
./virtualenv/bin/nosetests \
s3tests_boto3/functional/test_s3.py \
-v \
-a 'resource=object,!bucket-policy,!versioning,!encryption' \
| tee ${TEST_RAW_OUTPUT_FILE}
# If the summary logs are present, process them
if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
rm -rf ${TEST_RAW_OUTPUT_FILE};
fi
else
echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
fi
echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
docker stop $CONTAINER_NAME || echo "[info] already stopped";
echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
killall -9 weed;
+8
View File
@@ -4,6 +4,14 @@
# host set for rgw in vstart.sh
host = host.docker.internal
## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
## then use 'localhost', so the container can more easily access your *local* weed instance
##
## This is *unsafe* -- it is possible that this script will do malicious
## things with host-level access. only run this if you trust the programmatic content of
## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
#host = localhost
# port set for rgw in vstart.sh
port = 8333
+2 -2
View File
@@ -33,8 +33,8 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
return nil
}
+3 -3
View File
@@ -57,7 +57,7 @@ func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
lc: lc,
}
go func() {
util.RetryForever("create lock:"+key, func() error {
util.RetryUntil("create lock:"+key, func() error {
errorMessage, err := lock.doLock(lock_manager.MaxDuration)
if err != nil {
glog.Infof("create lock %s: %s", key, err)
@@ -98,7 +98,7 @@ func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner st
lockDuration = lc.maxLockDuration
needRenewal = true
}
util.RetryForever("create lock:"+key, func() error {
util.RetryUntil("create lock:"+key, func() error {
errorMessage, err := lock.doLock(lockDuration)
if err != nil {
time.Sleep(time.Second)
@@ -148,7 +148,7 @@ func (lc *LockClient) keepLock(lock *LiveLock) {
select {
case <-ticker:
// renew the lock if lock.expireAtNs is still greater than now
util.RetryForever("keep lock:"+lock.key, func() error {
util.RetryUntil("keep lock:"+lock.key, func() error {
lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
if lockDuration > lc.maxLockDuration {
lockDuration = lc.maxLockDuration
+1 -1
View File
@@ -169,7 +169,7 @@ func runFiler(cmd *Command, args []string) bool {
go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
filerAddress := util.JoinHostPort(*f.ip, *f.port)
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
startDelay := time.Duration(2)
if *filerStartS3 {
filerS3Options.filer = &filerAddress
+1 -1
View File
@@ -111,7 +111,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {
// synchronize /buckets folder
fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
util.RetryForever("filer.remote.sync buckets", func() error {
util.RetryUntil("filer.remote.sync buckets", func() error {
return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
}, func(err error) bool {
if err != nil {
+1 -1
View File
@@ -90,7 +90,7 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
if dir != "" {
fmt.Printf("synchronize %s to remote storage...\n", dir)
util.RetryForever("filer.remote.sync "+dir, func() error {
util.RetryUntil("filer.remote.sync "+dir, func() error {
return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
}, func(err error) bool {
if err != nil {
+1 -1
View File
@@ -99,7 +99,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
if m.OldEntry != nil && m.NewEntry == nil {
glog.V(1).Infof("delete: %s", key)
} else if m.OldEntry == nil && m.NewEntry != nil {
glog.V(1).Infof(" add: %s", key)
glog.V(1).Infof("add: %s", key)
} else {
glog.V(1).Infof("modify: %s", key)
}
+2 -2
View File
@@ -288,8 +288,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
}
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
return strings.Compare(string(a), string(b)) < 0
slices.SortFunc(peers, func(a, b pb.ServerAddress) int {
return strings.Compare(string(a), string(b))
})
if len(peers) <= 0 {
return true
+27 -1
View File
@@ -3,7 +3,9 @@ package command
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
@@ -42,6 +44,8 @@ type S3Options struct {
domainName *string
tlsPrivateKey *string
tlsCertificate *string
tlsCACertificate *string
tlsVerifyClientCert *bool
metricsHttpPort *int
allowEmptyFolder *bool
allowDeleteBucketNotEmpty *bool
@@ -65,6 +69,8 @@ func init() {
s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file")
s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate")
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
@@ -289,7 +295,27 @@ func (s3opt *S3Options) startS3Server() bool {
if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
}
httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
caCertPool := x509.NewCertPool()
if *s3Options.tlsCACertificate != "" {
// load CA certificate file and add it to list of client CAs
caCertFile, err := ioutil.ReadFile(*s3opt.tlsCACertificate)
if err != nil {
glog.Fatalf("error reading CA certificate: %v", err)
}
caCertPool.AppendCertsFromPEM(caCertFile)
}
clientAuth := tls.NoClientCert
if *s3Options.tlsVerifyClientCert {
clientAuth = tls.RequireAndVerifyClientCert
}
httpS.TLSConfig = &tls.Config{
GetCertificate: s3opt.GetCertificateWithUpdate,
ClientAuth: clientAuth,
ClientCAs: caCertPool,
}
if *s3opt.portHttps == 0 {
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
if s3ApiLocalListener != nil {
+1
View File
@@ -12,6 +12,7 @@
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
recursive_delete = false
#max_file_name_length = 255
####################################################
# The following are filer store options
+2
View File
@@ -144,6 +144,8 @@ func init() {
s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
s3Options.tlsCACertificate = cmdServer.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
s3Options.tlsVerifyClientCert = cmdServer.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
+2 -1
View File
@@ -19,6 +19,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/server/constants"
"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
"google.golang.org/grpc/reflection"
@@ -241,7 +242,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
*v.idxFolder,
volumeNeedleMapKind,
v.masters, 5, *v.dataCenter, *v.rack,
v.masters, constants.VolumePulseSeconds, *v.dataCenter, *v.rack,
v.whiteList,
*v.fixJpgOrientation, *v.readMode,
*v.compactionMBPerSecond,
+3 -3
View File
@@ -73,11 +73,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
}
func printChunks(name string, chunks []*filer_pb.FileChunk) {
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
if a.Offset == b.Offset {
return a.ModifiedTsNs < b.ModifiedTsNs
return int(a.ModifiedTsNs - b.ModifiedTsNs)
}
return a.Offset < b.Offset
return int(a.Offset - b.Offset)
})
for _, chunk := range chunks {
glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
+10 -4
View File
@@ -30,14 +30,20 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
isStart: false,
})
}
slices.SortFunc(points, func(a, b *Point) bool {
slices.SortFunc(points, func(a, b *Point) int {
if a.x != b.x {
return a.x < b.x
return int(a.x - b.x)
}
if a.ts != b.ts {
return a.ts < b.ts
return int(a.ts - b.ts)
}
return !a.isStart
if a.isStart {
return -1
}
if b.isStart {
return 1
}
return 0
})
var prevX int64
+8 -2
View File
@@ -50,9 +50,10 @@ type Filer struct {
FilerConf *FilerConf
RemoteStorage *FilerRemoteStorage
Dlm *lock_manager.DistributedLockManager
MaxFilenameLength uint32
}
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
f := &Filer{
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
fileIdDeletionQueue: util.NewUnboundedQueue(),
@@ -61,6 +62,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
RemoteStorage: NewFilerRemoteStorage(),
UniqueFilerId: util.RandomInt32(),
Dlm: lock_manager.NewDistributedLockManager(filerHost),
MaxFilenameLength: maxFilenameLength,
}
if f.UniqueFilerId < 0 {
f.UniqueFilerId = -f.UniqueFilerId
@@ -194,12 +196,16 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
return f.Store.RollbackTransaction(ctx)
}
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {
if string(entry.FullPath) == "/" {
return nil
}
if entry.FullPath.IsLongerFileName(maxFilenameLength) {
return fmt.Errorf("entry name too long")
}
oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
/*
+3
View File
@@ -166,6 +166,9 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
a.VolumeGrowthCount = b.VolumeGrowthCount
}
a.ReadOnly = b.ReadOnly || a.ReadOnly
if b.MaxFileNameLength > 0 {
a.MaxFileNameLength = b.MaxFileNameLength
}
a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter)
a.Rack = util.Nvl(b.Rack, a.Rack)
a.DataNode = util.Nvl(b.DataNode, a.DataNode)
+1 -1
View File
@@ -43,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset, time.Now().UnixNano()))
// update the entry
err = f.CreateEntry(context.Background(), entry, false, false, nil, false)
err = f.CreateEntry(context.Background(), entry, false, false, nil, false, f.MaxFilenameLength)
return err
}
+1 -1
View File
@@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
}
}
}
if count < limit && lastFileName <= prefix && len(notPrefixed) == int(limit) {
if count < limit && lastFileName <= prefix {
notPrefixed = notPrefixed[:0]
lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
notPrefixed = append(notPrefixed, entry)
+4 -4
View File
@@ -13,7 +13,7 @@ import (
)
func TestCreateAndFind(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDBStore{}
store.initialize(dir)
@@ -32,7 +32,7 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
@@ -66,7 +66,7 @@ func TestCreateAndFind(t *testing.T) {
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDBStore{}
store.initialize(dir)
@@ -88,7 +88,7 @@ func TestEmptyRoot(t *testing.T) {
}
func BenchmarkInsertEntry(b *testing.B) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := b.TempDir()
store := &LevelDBStore{}
store.initialize(dir)
+3 -3
View File
@@ -10,7 +10,7 @@ import (
)
func TestCreateAndFind(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDB2Store{}
store.initialize(dir, 2)
@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
@@ -63,7 +63,7 @@ func TestCreateAndFind(t *testing.T) {
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDB2Store{}
store.initialize(dir, 2)
+3 -3
View File
@@ -10,7 +10,7 @@ import (
)
func TestCreateAndFind(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDB3Store{}
store.initialize(dir)
@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
@@ -63,7 +63,7 @@ func TestCreateAndFind(t *testing.T) {
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
store := &LevelDB3Store{}
store.initialize(dir)
+2 -2
View File
@@ -164,8 +164,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
}
// sort
slices.SortFunc(members, func(a, b string) bool {
return strings.Compare(a, b) < 0
slices.SortFunc(members, func(a, b string) int {
return strings.Compare(a, b)
})
// limit
+1 -1
View File
@@ -34,7 +34,7 @@ func TestCreateAndFind(t *testing.T) {
},
}
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
+4 -4
View File
@@ -45,11 +45,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool {
if len(a) != len(b) {
return false
}
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) int {
return strings.Compare(i.ETag, j.ETag)
})
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
return strings.Compare(i.ETag, j.ETag) < 0
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) int {
return strings.Compare(i.ETag, j.ETag)
})
for i := 0; i < len(a); i++ {
if a[i].ETag != b[i].ETag {
+17 -50
View File
@@ -228,65 +228,32 @@ func (store *TikvStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPat
return err
}
defer iter.Close()
i := int64(0)
first := true
for iter.Valid() {
if first {
first = false
if !includeStartFile {
if iter.Valid() {
// Check first item is lastFileStart
if bytes.Equal(iter.Key(), lastFileStart) {
// Is lastFileStart and not include start file, just
// ignore it.
err = iter.Next()
if err != nil {
return err
}
continue
}
}
}
}
// Check for limitation
if limit > 0 {
i++
if i > limit {
break
}
}
// Validate key prefix
for i := int64(0); i < limit && iter.Valid(); i++ {
key := iter.Key()
if !bytes.HasPrefix(key, directoryPrefix) {
break
}
value := iter.Value()
// Start process
fileName := getNameFromKey(key)
if fileName != "" {
// Got file name, then generate the Entry
entry := &filer.Entry{
FullPath: util.NewFullPath(string(dirPath), fileName),
}
// Update lastFileName
lastFileName = fileName
// Check for decode value.
if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil {
// Got error just return the error
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
return err
}
// Run for each callback if return false just break the iteration
if !eachEntryFunc(entry) {
if fileName == "" || fileName == startFileName && !includeStartFile {
if err := iter.Next(); err != nil {
break
} else {
continue
}
}
// End process
lastFileName = fileName
entry := &filer.Entry{
FullPath: util.NewFullPath(string(dirPath), fileName),
}
err = iter.Next()
if err != nil {
return err
// println("list", entry.FullPath, "chunks", len(entry.GetChunks()))
if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(iter.Value())); decodeErr != nil {
err = decodeErr
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
break
}
if err := iter.Next(); !eachEntryFunc(entry) || err != nil {
break
}
}
return nil
+4 -1
View File
@@ -150,7 +150,10 @@ func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
defer i.RUnlock()
inode, found := i.path2inode[fullpath]
if !found {
glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
// https://github.com/seaweedfs/seaweedfs/issues/4968
// glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
glog.Warningf("MarkChildrenCached not found inode %v", fullpath)
return
}
path, found := i.inode2path[inode]
path.isChildrenCached = true
@@ -57,11 +57,10 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
}
var clientEpoch int32
metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "mount",
ClientId: selfSignature,
ClientEpoch: clientEpoch,
ClientEpoch: 1,
SelfSignature: selfSignature,
PathPrefix: dir,
AdditionalPathPrefixes: nil,
@@ -70,8 +69,8 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
StopTsNs: 0,
EventErrorType: pb.FatalOnError,
}
util.RetryForever("followMetaUpdates", func() error {
clientEpoch++
util.RetryUntil("followMetaUpdates", func() error {
metadataFollowOption.ClientEpoch++
return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, processEventFn)
}, func(err error) bool {
glog.Errorf("follow metadata updates: %v", err)
+1 -1
View File
@@ -43,7 +43,7 @@ func Test_allocateOneBroker(t *testing.T) {
type args struct {
brokers cmap.ConcurrentMap[string, *BrokerStats]
partitionCount int
partitionCount int32
}
func testThem(t *testing.T, tests []struct {
+35 -17
View File
@@ -3,6 +3,7 @@ package balancer
import (
"fmt"
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
)
@@ -11,6 +12,23 @@ const (
LockBrokerBalancer = "broker_balancer"
)
// Balancer collects stats from all brokers.
//
// When publishers wants to create topics, it picks brokers to assign the topic partitions.
// When consumers wants to subscribe topics, it tells which brokers are serving the topic partitions.
//
// When a partition needs to be split or merged, or a partition needs to be moved to another broker,
// the balancer will let the broker tell the consumer instance to stop processing the partition.
// The existing consumer instance will flush the internal state, and then stop processing.
// Then the balancer will tell the brokers to start sending new messages in the new/moved partition to the consumer instances.
//
// Failover to standby consumer instances:
//
// A consumer group can have min and max number of consumer instances.
// For consumer instances joined after the max number, they will be in standby mode.
//
// When a consumer instance is down, the broker will notice this and inform the balancer.
// The balancer will then tell the broker to send the partition to another standby consumer instance.
type Balancer struct {
Brokers cmap.ConcurrentMap[string, *BrokerStats]
}
@@ -30,11 +48,9 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
currentTopicPartitions := bs.Stats.Items()
for _, topicPartitionStats := range stats.Stats {
tps := &TopicPartitionStats{
TopicPartition: TopicPartition{
Namespace: topicPartitionStats.Topic.Namespace,
Topic: topicPartitionStats.Topic.Name,
RangeStart: topicPartitionStats.Partition.RangeStart,
RangeStop: topicPartitionStats.Partition.RangeStop,
TopicPartition: topic.TopicPartition{
Topic: topic.Topic{Namespace: topicPartitionStats.Topic.Namespace, Name: topicPartitionStats.Topic.Name},
Partition: topic.Partition{RangeStart: topicPartitionStats.Partition.RangeStart, RangeStop: topicPartitionStats.Partition.RangeStop, RingSize: topicPartitionStats.Partition.RingSize},
},
ConsumerCount: topicPartitionStats.ConsumerCount,
IsLeader: topicPartitionStats.IsLeader,
@@ -52,20 +68,26 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
}
func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Partition) {
tps := &TopicPartitionStats{
TopicPartition: topic.TopicPartition{
Topic: topic.Topic{Namespace: t.Namespace, Name: t.Name},
Partition: topic.Partition{RangeStart: partition.RangeStart, RangeStop: partition.RangeStop},
},
ConsumerCount: 0,
IsLeader: true,
}
key := tps.TopicPartition.String()
bs.Stats.Set(key, tps)
}
func (bs *BrokerStats) String() string {
return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}",
bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.Stats.Items())
}
type TopicPartition struct {
Namespace string
Topic string
RangeStart int32
RangeStop int32
}
type TopicPartitionStats struct {
TopicPartition
topic.TopicPartition
ConsumerCount int32
IsLeader bool
}
@@ -81,7 +103,3 @@ func NewBrokerStats() *BrokerStats {
Stats: cmap.New[*TopicPartitionStats](),
}
}
func (tp *TopicPartition) String() string {
return fmt.Sprintf("%v.%v-%04d-%04d", tp.Namespace, tp.Topic, tp.RangeStart, tp.RangeStop)
}
+3 -6
View File
@@ -16,7 +16,7 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
topicPartitionStat := topicPartitionStatsItem.Val
if topicPartitionStat.TopicPartition.Namespace == topic.Namespace &&
topicPartitionStat.TopicPartition.Topic == topic.Name {
topicPartitionStat.TopicPartition.Name == topic.Name {
assignment := &mq_pb.BrokerPartitionAssignment{
Partition: &mq_pb.Partition{
RingSize: MaxPartitionCount,
@@ -24,11 +24,8 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
RangeStop: topicPartitionStat.RangeStop,
},
}
if topicPartitionStat.IsLeader {
assignment.LeaderBroker = broker
} else {
assignment.FollowerBrokers = append(assignment.FollowerBrokers, broker)
}
// TODO fix follower setting
assignment.LeaderBroker = broker
assignments = append(assignments, assignment)
}
}
+12 -5
View File
@@ -8,7 +8,7 @@ import (
"google.golang.org/grpc/status"
)
// BrokerConnectToBalancer receives connections from brokers and collects stats
// ConnectToBalancer receives connections from brokers and collects stats
func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessaging_ConnectToBalancerServer) error {
if !broker.lockAsBalancer.IsLocked() {
return status.Errorf(codes.Unavailable, "not current broker balancer")
@@ -20,9 +20,16 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
// process init message
initMessage := req.GetInit()
brokerStats := balancer.NewBrokerStats()
var brokerStats *balancer.BrokerStats
if initMessage != nil {
broker.Balancer.Brokers.Set(initMessage.Broker, brokerStats)
var found bool
brokerStats, found = broker.Balancer.Brokers.Get(initMessage.Broker)
if !found {
brokerStats = balancer.NewBrokerStats()
if !broker.Balancer.Brokers.SetIfAbsent(initMessage.Broker, brokerStats) {
brokerStats, _ = broker.Balancer.Brokers.Get(initMessage.Broker)
}
}
} else {
return status.Errorf(codes.InvalidArgument, "balancer init message is empty")
}
@@ -42,8 +49,8 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
if receivedStats := req.GetStats(); receivedStats != nil {
brokerStats.UpdateStats(receivedStats)
glog.V(3).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
glog.V(3).Infof("received stats: %+v", receivedStats)
glog.V(4).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
glog.V(4).Infof("received stats: %+v", receivedStats)
}
}
+24 -8
View File
@@ -2,6 +2,8 @@ package broker
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/mq/balancer"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -9,14 +11,14 @@ import (
"google.golang.org/grpc/status"
)
// CreateTopic Runs on any broker, but proxied to the balancer if not the balancer
func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_pb.CreateTopicRequest) (resp *mq_pb.CreateTopicResponse, err error) {
// ConfigureTopic Runs on any broker, but proxied to the balancer if not the balancer
func (broker *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.ConfigureTopicRequest) (resp *mq_pb.ConfigureTopicResponse, err error) {
if broker.currentBalancer == "" {
return nil, status.Errorf(codes.Unavailable, "no balancer")
}
if !broker.lockAsBalancer.IsLocked() {
proxyErr := broker.withBrokerClient(false, broker.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
resp, err = client.CreateTopic(ctx, request)
resp, err = client.ConfigureTopic(ctx, request)
return nil
})
if proxyErr != nil {
@@ -25,26 +27,40 @@ func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_p
return resp, err
}
ret := &mq_pb.CreateTopicResponse{}
ret := &mq_pb.ConfigureTopicResponse{}
ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
for _, bpa := range ret.BrokerPartitionAssignments {
// fmt.Printf("create topic %s on %s\n", request.Topic, bpa.LeaderBroker)
if doCreateErr := broker.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
_, doCreateErr := client.DoCreateTopic(ctx, &mq_pb.DoCreateTopicRequest{
_, doCreateErr := client.DoConfigureTopic(ctx, &mq_pb.DoConfigureTopicRequest{
Topic: request.Topic,
Partition: bpa.Partition,
})
return doCreateErr
if doCreateErr != nil {
return fmt.Errorf("do create topic %s on %s: %v", request.Topic, bpa.LeaderBroker, doCreateErr)
}
brokerStats, found := broker.Balancer.Brokers.Get(bpa.LeaderBroker)
if !found {
brokerStats = balancer.NewBrokerStats()
if !broker.Balancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
brokerStats, _ = broker.Balancer.Brokers.Get(bpa.LeaderBroker)
}
}
brokerStats.RegisterAssignment(request.Topic, bpa.Partition)
return nil
}); doCreateErr != nil {
return nil, doCreateErr
}
}
// TODO revert if some error happens in the middle of the assignments
return ret, err
}
func (broker *MessageQueueBroker) DoCreateTopic(ctx context.Context, req *mq_pb.DoCreateTopicRequest) (resp *mq_pb.DoCreateTopicResponse, err error) {
ret := &mq_pb.DoCreateTopicResponse{}
func (broker *MessageQueueBroker) DoConfigureTopic(ctx context.Context, req *mq_pb.DoConfigureTopicRequest) (resp *mq_pb.DoConfigureTopicResponse, err error) {
ret := &mq_pb.DoConfigureTopicResponse{}
t, p := topic.FromPbTopic(req.Topic), topic.FromPbPartition(req.Partition)
localTopicPartition := broker.localTopicManager.GetTopicPartition(t, p)
if localTopicPartition == nil {
+6 -3
View File
@@ -2,6 +2,7 @@ package broker
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -62,18 +63,20 @@ func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb
}
ret := &mq_pb.ListTopicsResponse{}
knownTopics := make(map[*mq_pb.Topic]struct{})
knownTopics := make(map[string]struct{})
for brokerStatsItem := range broker.Balancer.Brokers.IterBuffered() {
_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
topicPartitionStat := topicPartitionStatsItem.Val
topic := &mq_pb.Topic{
Namespace: topicPartitionStat.TopicPartition.Namespace,
Name: topicPartitionStat.TopicPartition.Topic,
Name: topicPartitionStat.TopicPartition.Name,
}
if _, found := knownTopics[topic]; found {
topicKey := fmt.Sprintf("%s/%s", topic.Namespace, topic.Name)
if _, found := knownTopics[topicKey]; found {
continue
}
knownTopics[topicKey] = struct{}{}
ret.Topics = append(ret.Topics, topic)
}
}
+3 -3
View File
@@ -11,8 +11,8 @@ import (
func (broker *MessageQueueBroker) Subscribe(req *mq_pb.SubscribeRequest, stream mq_pb.SeaweedMessaging_SubscribeServer) error {
localTopicPartition := broker.localTopicManager.GetTopicPartition(topic.FromPbTopic(req.Cursor.Topic),
topic.FromPbPartition(req.Cursor.Partition))
localTopicPartition := broker.localTopicManager.GetTopicPartition(topic.FromPbTopic(req.GetInit().Topic),
topic.FromPbPartition(req.GetInit().Partition))
if localTopicPartition == nil {
stream.Send(&mq_pb.SubscribeResponse{
Message: &mq_pb.SubscribeResponse_Ctrl{
@@ -24,7 +24,7 @@ func (broker *MessageQueueBroker) Subscribe(req *mq_pb.SubscribeRequest, stream
return nil
}
clientName := fmt.Sprintf("%s/%s-%s", req.Consumer.ConsumerGroup, req.Consumer.ConsumerId, req.Consumer.ClientId)
clientName := fmt.Sprintf("%s/%s-%s", req.GetInit().ConsumerGroup, req.GetInit().ConsumerId, req.GetInit().ClientId)
localTopicPartition.Subscribe(clientName, time.Now(), func(logEntry *filer_pb.LogEntry) error {
value := logEntry.GetData()
+1
View File
@@ -61,6 +61,7 @@ func (broker *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
if err != nil {
return fmt.Errorf("send stats message: %v", err)
}
glog.V(3).Infof("sent stats: %+v", stats)
time.Sleep(time.Millisecond*5000 + time.Duration(rand.Intn(1000))*time.Millisecond)
}
+47
View File
@@ -0,0 +1,47 @@
package broker
import (
"context"
"fmt"
"net"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/test/bufconn"
)
var lis *bufconn.Listener
func init() {
lis = bufconn.Listen(1024 * 1024)
server := grpc.NewServer()
mq_pb.RegisterSeaweedMessagingServer(server, &MessageQueueBroker{})
go func() {
if err := server.Serve(lis); err != nil {
fmt.Printf("Server exited with error: %v", err)
}
}()
}
func bufDialer(string, time.Duration) (net.Conn, error) {
return lis.Dial()
}
func TestMessageQueueBroker_ListTopics(t *testing.T) {
conn, err := grpc.DialContext(context.Background(), "bufnet", grpc.WithDialer(bufDialer), grpc.WithInsecure())
if err != nil {
t.Fatalf("Failed to dial bufnet: %v", err)
}
defer conn.Close()
client := mq_pb.NewSeaweedMessagingClient(conn)
request := &mq_pb.ListTopicsRequest{}
_, err = client.ListTopics(context.Background(), request)
if err == nil {
t.Fatalf("Add failed: %v", err)
}
}
+1 -5
View File
@@ -22,11 +22,7 @@ func main() {
Filter: "",
}
subscriber := sub_client.NewTopicSubscriber(subscriberConfig, contentConfig)
if err := subscriber.Connect("localhost:17777"); err != nil {
fmt.Println(err)
return
}
subscriber := sub_client.NewTopicSubscriber("localhost:17777", subscriberConfig, contentConfig)
subscriber.SetEachMessageFunc(func(key, value []byte) bool {
println(string(key), "=>", string(value))
+81
View File
@@ -0,0 +1,81 @@
package sub_client
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"sync"
)
func (sub *TopicSubscriber) doProcess() error {
var wg sync.WaitGroup
for _, brokerPartitionAssignment := range sub.brokerPartitionAssignments {
brokerAddress := brokerPartitionAssignment.LeaderBroker
grpcConnection, err := pb.GrpcDial(context.Background(), brokerAddress, true, sub.SubscriberConfig.GrpcDialOption)
if err != nil {
return fmt.Errorf("dial broker %s: %v", brokerAddress, err)
}
brokerClient := mq_pb.NewSeaweedMessagingClient(grpcConnection)
subscribeClient, err := brokerClient.Subscribe(context.Background(), &mq_pb.SubscribeRequest{
Message: &mq_pb.SubscribeRequest_Init{
Init: &mq_pb.SubscribeRequest_InitMessage{
ConsumerGroup: sub.SubscriberConfig.GroupId,
ConsumerId: sub.SubscriberConfig.GroupInstanceId,
Topic: &mq_pb.Topic{
Namespace: sub.ContentConfig.Namespace,
Name: sub.ContentConfig.Topic,
},
Partition: &mq_pb.Partition{
RingSize: brokerPartitionAssignment.Partition.RingSize,
RangeStart: brokerPartitionAssignment.Partition.RangeStart,
RangeStop: brokerPartitionAssignment.Partition.RangeStop,
},
Filter: sub.ContentConfig.Filter,
},
},
})
if err != nil {
return fmt.Errorf("create subscribe client: %v", err)
}
wg.Add(1)
go func() {
defer wg.Done()
if sub.OnCompletionFunc != nil {
defer sub.OnCompletionFunc()
}
defer func() {
subscribeClient.SendMsg(&mq_pb.SubscribeRequest{
Message: &mq_pb.SubscribeRequest_Ack{
Ack: &mq_pb.SubscribeRequest_AckMessage{
Sequence: 0,
},
},
})
subscribeClient.CloseSend()
}()
for {
resp, err := subscribeClient.Recv()
if err != nil {
fmt.Printf("subscribe error: %v\n", err)
return
}
if resp.Message == nil {
continue
}
switch m := resp.Message.(type) {
case *mq_pb.SubscribeResponse_Data:
if !sub.OnEachMessageFunc(m.Data.Key, m.Data.Value) {
return
}
case *mq_pb.SubscribeResponse_Ctrl:
if m.Ctrl.IsEndOfStream || m.Ctrl.IsEndOfTopic {
return
}
}
}
}()
}
wg.Wait()
return nil
}
+19 -58
View File
@@ -1,72 +1,33 @@
package sub_client
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"sync"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
)
// Subscribe subscribes to a topic's specified partitions.
// If a partition is moved to another broker, the subscriber will automatically reconnect to the new broker.
func (sub *TopicSubscriber) Subscribe() error {
var wg sync.WaitGroup
for _, brokerPartitionAssignment := range sub.brokerPartitionAssignments {
brokerAddress := brokerPartitionAssignment.LeaderBroker
grpcConnection, err := pb.GrpcDial(context.Background(), brokerAddress, true, sub.SubscriberConfig.GrpcDialOption)
if err != nil {
return fmt.Errorf("dial broker %s: %v", brokerAddress, err)
util.RetryUntil("subscribe", func() error {
// ask balancer for brokers of the topic
if err := sub.doLookup(sub.bootstrapBroker); err != nil {
return fmt.Errorf("lookup topic %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
}
brokerClient := mq_pb.NewSeaweedMessagingClient(grpcConnection)
subscribeClient, err := brokerClient.Subscribe(context.Background(), &mq_pb.SubscribeRequest{
Consumer: &mq_pb.SubscribeRequest_Consumer{
ConsumerGroup: sub.SubscriberConfig.GroupId,
ConsumerId: sub.SubscriberConfig.GroupInstanceId,
},
Cursor: &mq_pb.SubscribeRequest_Cursor{
Topic: &mq_pb.Topic{
Namespace: sub.ContentConfig.Namespace,
Name: sub.ContentConfig.Topic,
},
Partition: &mq_pb.Partition{
RingSize: brokerPartitionAssignment.Partition.RingSize,
RangeStart: brokerPartitionAssignment.Partition.RangeStart,
RangeStop: brokerPartitionAssignment.Partition.RangeStop,
},
Filter: sub.ContentConfig.Filter,
},
})
if err != nil {
return fmt.Errorf("create subscribe client: %v", err)
// treat the first broker as the topic leader
// connect to the leader broker
// subscribe to the topic
if err := sub.doProcess(); err != nil {
return fmt.Errorf("subscribe topic %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
}
wg.Add(1)
go func() {
defer wg.Done()
if sub.OnCompletionFunc != nil {
defer sub.OnCompletionFunc()
}
for {
resp, err := subscribeClient.Recv()
if err != nil {
fmt.Printf("subscribe error: %v\n", err)
return
}
if resp.Message == nil {
continue
}
switch m := resp.Message.(type) {
case *mq_pb.SubscribeResponse_Data:
if !sub.OnEachMessageFunc(m.Data.Key, m.Data.Value) {
return
}
case *mq_pb.SubscribeResponse_Ctrl:
// ignore
}
}
}()
}
wg.Wait()
return nil
}, func(err error) bool {
if err == io.EOF {
return false
}
return true
})
return nil
}
+10 -13
View File
@@ -6,11 +6,13 @@ import (
)
type SubscriberConfiguration struct {
ClientId string
GroupId string
GroupInstanceId string
BootstrapServers []string
GrpcDialOption grpc.DialOption
ClientId string
GroupId string
GroupInstanceId string
GroupMinimumPeers int32
GroupMaximumPeers int32
BootstrapServers []string
GrpcDialOption grpc.DialOption
}
type ContentConfiguration struct {
@@ -28,22 +30,17 @@ type TopicSubscriber struct {
brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment
OnEachMessageFunc OnEachMessageFunc
OnCompletionFunc OnCompletionFunc
bootstrapBroker string
}
func NewTopicSubscriber(subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
func NewTopicSubscriber(bootstrapBroker string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
return &TopicSubscriber{
SubscriberConfig: subscriber,
ContentConfig: content,
bootstrapBroker: bootstrapBroker,
}
}
func (sub *TopicSubscriber) Connect(bootstrapBroker string) error {
if err := sub.doLookup(bootstrapBroker); err != nil {
return err
}
return nil
}
func (sub *TopicSubscriber) SetEachMessageFunc(onEachMessageFn OnEachMessageFunc) {
sub.OnEachMessageFunc = onEachMessageFn
}
+92
View File
@@ -0,0 +1,92 @@
package coordinator
import (
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"sync"
)
func (cg *ConsumerGroup) SetMinMaxActiveInstances(min, max int32) {
cg.MinimumActiveInstances = min
cg.MaximumActiveInstances = max
}
func (cg *ConsumerGroup) AddConsumerGroupInstance(clientId string) *ConsumerGroupInstance {
cgi := &ConsumerGroupInstance{
ClientId: clientId,
}
cg.ConsumerGroupInstances.Set(clientId, cgi)
return cgi
}
func (cg *ConsumerGroup) RemoveConsumerGroupInstance(clientId string) {
cg.ConsumerGroupInstances.Remove(clientId)
}
func (cg *ConsumerGroup) CoordinateIfNeeded() {
emptyInstanceCount, activeInstanceCount := int32(0), int32(0)
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
if cgi.Val.Partition == nil {
// this consumer group instance is not assigned a partition
// need to assign one
emptyInstanceCount++
} else {
activeInstanceCount++
}
}
var delta int32
if emptyInstanceCount > 0 {
if cg.MinimumActiveInstances <= 0 {
// need to assign more partitions
delta = emptyInstanceCount
} else if activeInstanceCount < cg.MinimumActiveInstances && activeInstanceCount+emptyInstanceCount >= cg.MinimumActiveInstances {
// need to assign more partitions
delta = cg.MinimumActiveInstances - activeInstanceCount
}
}
if cg.MaximumActiveInstances > 0 {
if activeInstanceCount > cg.MaximumActiveInstances {
// need to remove some partitions
delta = cg.MaximumActiveInstances - activeInstanceCount
}
}
if delta == 0 {
return
}
cg.doCoordinate(activeInstanceCount + delta)
}
func (cg *ConsumerGroup) doCoordinate(target int32) {
// stop existing instances from processing
var wg sync.WaitGroup
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
if cgi.Val.Partition != nil {
wg.Add(1)
go func(cgi *ConsumerGroupInstance) {
defer wg.Done()
// stop processing
// flush internal state
// wait for all messages to be processed
// close the connection
}(cgi.Val)
}
}
wg.Wait()
partitions := topic.SplitPartitions(target)
// assign partitions to new instances
i := 0
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
cgi.Val.Partition = partitions[i]
i++
wg.Add(1)
go func(cgi *ConsumerGroupInstance) {
defer wg.Done()
// start processing
// start consuming from the last offset
}(cgi.Val)
}
wg.Wait()
}
+36
View File
@@ -0,0 +1,36 @@
package coordinator
import (
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
)
type ConsumerGroupInstance struct {
ClientId string
// the consumer group instance may not have an active partition
Partition *topic.Partition
// processed message count
ProcessedMessageCount int64
}
type ConsumerGroup struct {
// map a client id to a consumer group instance
ConsumerGroupInstances cmap.ConcurrentMap[string, *ConsumerGroupInstance]
MinimumActiveInstances int32
MaximumActiveInstances int32
}
type TopicConsumerGroups struct {
// map a consumer group name to a consumer group
ConsumerGroups cmap.ConcurrentMap[string, *ConsumerGroup]
}
// Coordinator coordinates the instances in the consumer group for one topic.
// It is responsible for:
// 1. Assigning partitions to consumer instances.
// 2. Reassigning partitions when a consumer instance is down.
// 3. Reassigning partitions when a consumer instance is up.
type Coordinator struct {
// map client id to subscriber
Subscribers cmap.ConcurrentMap[string, *ConsumerGroupInstance]
// map topic name to consumer groups
TopicSubscribers cmap.ConcurrentMap[string, map[string]TopicConsumerGroups]
}
+23 -9
View File
@@ -19,7 +19,7 @@ func NewLocalTopicManager() *LocalTopicManager {
}
}
// AddTopic adds a topic to the local topic manager
// AddTopicPartition adds a topic to the local topic manager
func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition *LocalPartition) {
localTopic, ok := manager.topics.Get(topic.String())
if !ok {
@@ -28,7 +28,9 @@ func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition
Partitions: make([]*LocalPartition, 0),
}
}
manager.topics.SetIfAbsent(topic.String(), localTopic)
if !manager.topics.SetIfAbsent(topic.String(), localTopic) {
localTopic, _ = manager.topics.Get(topic.String())
}
if localTopic.findPartition(localPartition.Partition) != nil {
return
}
@@ -61,9 +63,26 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
stats := &mq_pb.BrokerStats{
Stats: make(map[string]*mq_pb.TopicPartitionStats),
}
// collect current broker's cpu usage
// this needs to be in front, so the following stats can be more accurate
usages, err := cpu.Percent(duration, false)
if err == nil && len(usages) > 0 {
stats.CpuUsagePercent = int32(usages[0])
}
// collect current broker's topics and partitions
manager.topics.IterCb(func(topic string, localTopic *LocalTopic) {
for _, localPartition := range localTopic.Partitions {
stats.Stats[topic] = &mq_pb.TopicPartitionStats{
topicPartition := &TopicPartition{
Topic: Topic{Namespace: localTopic.Namespace, Name: localTopic.Name},
Partition: Partition{
RingSize: localPartition.RingSize,
RangeStart: localPartition.RangeStart,
RangeStop: localPartition.RangeStop,
},
}
stats.Stats[topicPartition.String()] = &mq_pb.TopicPartitionStats{
Topic: &mq_pb.Topic{
Namespace: string(localTopic.Namespace),
Name: localTopic.Name,
@@ -75,15 +94,10 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
},
ConsumerCount: localPartition.ConsumerCount,
}
// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
}
})
// collect current broker's cpu usage
usages, err := cpu.Percent(duration, false)
if err == nil && len(usages) > 0 {
stats.CpuUsagePercent = int32(usages[0])
}
return stats
}
+17
View File
@@ -30,3 +30,20 @@ func FromPbPartition(partition *mq_pb.Partition) Partition {
RingSize: partition.RingSize,
}
}
func SplitPartitions(targetCount int32) []*Partition {
partitions := make([]*Partition, 0, targetCount)
partitionSize := PartitionCount / targetCount
for i := int32(0); i < targetCount; i++ {
partitionStop := (i + 1) * partitionSize
if i == targetCount-1 {
partitionStop = PartitionCount
}
partitions = append(partitions, &Partition{
RangeStart: i * partitionSize,
RangeStop: partitionStop,
RingSize: PartitionCount,
})
}
return partitions
}
+4 -6
View File
@@ -7,14 +7,12 @@ import (
"time"
)
type Namespace string
type Topic struct {
Namespace Namespace
Namespace string
Name string
}
func NewTopic(namespace Namespace, name string) Topic {
func NewTopic(namespace string, name string) Topic {
return Topic{
Namespace: namespace,
Name: name,
@@ -22,7 +20,7 @@ func NewTopic(namespace Namespace, name string) Topic {
}
func FromPbTopic(topic *mq_pb.Topic) Topic {
return Topic{
Namespace: Namespace(topic.Namespace),
Namespace: topic.Namespace,
Name: topic.Name,
}
}
@@ -41,7 +39,7 @@ type Segment struct {
func FromPbSegment(segment *mq_pb.Segment) *Segment {
return &Segment{
Topic: Topic{
Namespace: Namespace(segment.Namespace),
Namespace: segment.Namespace,
Name: segment.Topic,
},
Id: segment.Id,
Loaded 100 of 191 files, more files were not shown because too many files have changed in this diff. Show more