Compare commits

...
Author SHA1 Message Date
chrislu 5b43c4bb98 3.44 2023-03-19 20:27:55 -07:00
chrislu 50dc2fe96b cleaning variables 2023-03-19 18:48:40 -07:00
29d1312625 Fs.verify.concurrency (#4293)
* init

* refactor

* refactor print msg

* param just concurrency

* fix race

* next fix race

* reset variables before each run

* clean

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2023-03-19 18:38:57 -07:00
chrislu 784daea1fa fix volume not found if marked as read only
fix https://github.com/seaweedfs/seaweedfs/issues/4088
2023-03-19 18:30:13 -07:00
Stewart Miles 264be0d2d4 Retry until a leader is selected. (#4318)
Fixes regression introduced in
https://github.com/seaweedfs/seaweedfs/pull/4313

Related to #4307
2023-03-16 20:50:38 -07:00
LHHDZ d695119073 The filer does not set defaultReplication to the defaultReplication o… (#4315)
The filer does not set defaultReplication to the defaultReplication of the master (it is not necessary, because if the filer is not set, the configuration of the master will be used when calling Assign), otherwise, when the defaultReplication of the master is modified and all master nodes are restarted, the defaultReplication will not take effect (because the filer Keep the previous defaultReplication setting from the master)

Signed-off-by: changlin.shi <changlin.shi@ly.com>
2023-03-15 22:10:39 -07:00
Stewart Miles 57ab1f8516 Use exponential backoff to query leader. (#4313)
`topology.Leader()` was using a backoff that typically
resulted in at least a 5s delay when initially starting
a master and raft server. This changes the backoff
algorithm to use exponential backoff starting with 100ms
and waiting up to 20s for leader selection.

Related to #4307
2023-03-15 17:49:46 -07:00
Stewart Miles dd71f54c6b Fix -raftHashicorp and -raftBootstrap flag propagation. (#4309)
`weed server` was not correctly propagating
`-master.raftHashicorp` and `-master.raftBootstrap` flags when
starting the master server.

Related to #4307
2023-03-15 13:03:20 -07:00
Patrick Schmidt 71b33faef0 Skip parent directory creation in mount (#4310)
A POSIX filesystem does not implicitly create parent directories when
they do not exist. Directories must be explicitly created and
permissions be set.

This also fixes a bug where asynchronous operations would create a
file in the filer before the parent directory was created. If the
file was a symlink or another special type of file the directory
would inherit that type and become unusable in the mounted FS.
2023-03-15 12:37:25 -07:00
chrislu f5854d13df fix test 2023-03-14 22:05:16 -07:00
chrislu 0454bb2d88 fix test
when size is zero, the "LastModified" is not written
2023-03-14 22:02:49 -07:00
chrislu 2268d38263 fix naming
fix https://github.com/seaweedfs/seaweedfs/issues/4305
2023-03-14 08:38:02 -07:00
dependabot[bot] b12d9b8598 build(deps): bump golang.org/x/image from 0.5.0 to 0.6.0 (#4304)
Bumps [golang.org/x/image](https://github.com/golang/image) from 0.5.0 to 0.6.0.
- [Release notes](https://github.com/golang/image/releases)
- [Commits](https://github.com/golang/image/compare/v0.5.0...v0.6.0)

---
updated-dependencies:
- dependency-name: golang.org/x/image
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-13 10:19:22 -07:00
dependabot[bot] 99c05a9c29 build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.42.12 to 3.43.0 (#4303)
build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3

Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.42.12 to 3.43.0.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.42.12...v3.43.0)

---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-13 10:01:26 -07:00
dependabot[bot] 4d3a485992 build(deps): bump google.golang.org/api from 0.111.0 to 0.112.0 (#4302)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.111.0 to 0.112.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.111.0...v0.112.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-13 10:01:12 -07:00
dependabot[bot] b1cdea3503 build(deps): bump github.com/aws/aws-sdk-go from 1.44.209 to 1.44.219 (#4301)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.209 to 1.44.219.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.209...v1.44.219)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-13 10:00:59 -07:00
dependabot[bot] 2b0cb47a43 build(deps): bump github.com/arangodb/go-driver from 1.4.1 to 1.5.2 (#4300)
Bumps [github.com/arangodb/go-driver](https://github.com/arangodb/go-driver) from 1.4.1 to 1.5.2.
- [Release notes](https://github.com/arangodb/go-driver/releases)
- [Changelog](https://github.com/arangodb/go-driver/blob/master/CHANGELOG.md)
- [Commits](https://github.com/arangodb/go-driver/compare/v1.4.1...v1.5.2)

---
updated-dependencies:
- dependency-name: github.com/arangodb/go-driver
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-13 10:00:44 -07:00
chrislu 98bc93cf31 fix tests 2023-03-13 10:00:27 -07:00
dependabot[bot] 66dd114933 build(deps): bump docker/setup-buildx-action from 2.4.1 to 2.5.0 (#4299) 2023-03-13 09:03:16 -07:00
Konstantin Lebedev e17429223e shell script unclean variables (#4298) 2023-03-13 07:16:31 -07:00
Thomas Anderson bd70683871 fix(weed/topology/node.go): typo in error message (#4292) 2023-03-08 23:58:03 -08:00
Rohit Chormale 59706c89fb #4270 set http status code to 409 if dir already exists (#4287) 2023-03-08 07:45:08 -08:00
chrislu 7111c08289 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2023-03-08 01:48:30 -08:00
chrislu c589e9837c fix tests 2023-03-08 01:48:28 -08:00
Kevin LiuandChris Lu 244385bf0d Fix binding metrics to ipv6 (#4286)
* Fix binding metrics to ipv6

* Update weed/stats/metrics.go

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2023-03-08 01:04:02 -08:00
Matt 889ecf5c9d added secret envs to filer (#4284) 2023-03-07 13:18:41 -08:00
dependabot[bot] 0cf4f2e891 build(deps): bump gocloud.dev/pubsub/rabbitpubsub from 0.28.0 to 0.29.0 (#4280)
Bumps [gocloud.dev/pubsub/rabbitpubsub](https://github.com/google/go-cloud) from 0.28.0 to 0.29.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.28.0...v0.29.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/rabbitpubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 10:49:44 -08:00
dependabot[bot] 113f3b76f4 build(deps): bump modernc.org/sqlite from 1.20.4 to 1.21.0 (#4277)
---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 09:46:45 -08:00
dependabot[bot] 9ed7922bed build(deps): bump golang.org/x/image from 0.0.0-20200119044424-58c23975cae1 to 0.5.0 (#4282)
build(deps): bump golang.org/x/image

Bumps [golang.org/x/image](https://github.com/golang/image) from 0.0.0-20200119044424-58c23975cae1 to 0.5.0.
- [Release notes](https://github.com/golang/image/releases)
- [Commits](https://github.com/golang/image/commits/v0.5.0)

---
updated-dependencies:
- dependency-name: golang.org/x/image
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 09:46:28 -08:00
dependabot[bot] e52bb3112d build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.42.10 to 3.42.12 (#4278)
build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3

Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.42.10 to 3.42.12.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.42.10...v3.42.12)

---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 09:27:21 -08:00
dependabot[bot] 281f5ef6a5 build(deps): bump golang.org/x/sys from 0.5.0 to 0.6.0 (#4279)
Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.5.0 to 0.6.0.
- [Release notes](https://github.com/golang/sys/releases)
- [Commits](https://github.com/golang/sys/compare/v0.5.0...v0.6.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sys
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 09:27:11 -08:00
dependabot[bot] c5b8a0554a build(deps): bump google.golang.org/api from 0.110.0 to 0.111.0 (#4281)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.110.0 to 0.111.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.110.0...v0.111.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-06 09:26:50 -08:00
chrislu cef493dd0d Update README.md 2023-03-02 08:38:32 -08:00
zemulandzemul bd6f2716d3 fix_read_tomb_needle_meta (#4263)
Co-authored-by: zemul <zhouzemiao@ihuman.com>
2023-03-02 08:26:55 -08:00
zemulandzemul 0bf56298d5 fix chunk.ModifiedTsNs (#4264)
* fix

* fix mtime s > ns

---------

Co-authored-by: zemul <zhouzemiao@ihuman.com>
2023-03-02 08:24:36 -08:00
chrislu 8241c9a829 adjust ui pagination size
fix https://github.com/seaweedfs/seaweedfs/issues/4258
2023-02-28 08:31:26 -08:00
chrislu 1f7934a89c enable gocdk in the generated local image
fix https://github.com/seaweedfs/seaweedfs/issues/4256
2023-02-27 19:20:02 -08:00
Matt 6cf43f9f58 Official Artifact Hub Metadata Resources (#4257)
* compatibility patch for csi driver

* added namespace to all component parameters

* added namespace to all component parameters

* dereference in range

* added namespace to values.yml defaults

* added namespace to s3 component

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* push on all tags

* push on all tags

* push on all tags

* push on all tags

* push on all tags

* changed helm directory structure

* update charts location

* fixed dereference

* updated permissions

* updated permissions

* match current action schema

* added helm chart liniting CI

* modified chart list changed

* fixed nodejs warning

* standardized a few defaults

* added chart dirs

* lowerd period seconds for volume startup to test chart lint changes

* lowerd period seconds for volume startup to test chart lint changes

* test

* changed back

* adjustment

* debug ls statment

* removed change detection

* remvoed change detection

* always lint the charts

* added missing )

* fixed typo

* added spaces in front of all comments

* pdated values.yaml

* pdated values.yaml

* don't validate maintainers for now

* update helm_ci.yml

* update helm_ci.yml

* update helm_ci.yml

* update helm_ci.yml

* update helm_ci.yml

* - adds helm chart lint on changes
- adds test helm chart install on k8s

* updated helm chart readme.md

* added artifact hub

* added artifact hub
2023-02-27 18:58:40 -08:00
dependabot[bot] 8cf75a152b build(deps): bump helm/kind-action from 1.4.0 to 1.5.0 (#4250)
Bumps [helm/kind-action](https://github.com/helm/kind-action) from 1.4.0 to 1.5.0.
- [Release notes](https://github.com/helm/kind-action/releases)
- [Commits](https://github.com/helm/kind-action/compare/v1.4.0...v1.5.0)

---
updated-dependencies:
- dependency-name: helm/kind-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:42:43 -08:00
dependabot[bot] 9db15c28ec build(deps): bump github.com/linxGnu/grocksdb from 1.7.14 to 1.7.15 (#4251)
Bumps [github.com/linxGnu/grocksdb](https://github.com/linxGnu/grocksdb) from 1.7.14 to 1.7.15.
- [Release notes](https://github.com/linxGnu/grocksdb/releases)
- [Commits](https://github.com/linxGnu/grocksdb/compare/v1.7.14...v1.7.15)

---
updated-dependencies:
- dependency-name: github.com/linxGnu/grocksdb
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:42:33 -08:00
dependabot[bot] 83d15b50ce build(deps): bump github.com/aws/aws-sdk-go from 1.44.189 to 1.44.209 (#4252)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.189 to 1.44.209.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.189...v1.44.209)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:42:21 -08:00
dependabot[bot] 32862b815b build(deps): bump github.com/hanwen/go-fuse/v2 from 2.1.1-0.20220627082937-d01fda7edf17 to 2.2.0 (#4253)
build(deps): bump github.com/hanwen/go-fuse/v2

Bumps [github.com/hanwen/go-fuse/v2](https://github.com/hanwen/go-fuse) from 2.1.1-0.20220627082937-d01fda7edf17 to 2.2.0.
- [Release notes](https://github.com/hanwen/go-fuse/releases)
- [Commits](https://github.com/hanwen/go-fuse/commits/v2.2.0)

---
updated-dependencies:
- dependency-name: github.com/hanwen/go-fuse/v2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:42:09 -08:00
dependabot[bot] c5bcfd8942 build(deps): bump gocloud.dev/pubsub/rabbitpubsub from 0.27.0 to 0.28.0 (#4254)
Bumps [gocloud.dev/pubsub/rabbitpubsub](https://github.com/google/go-cloud) from 0.27.0 to 0.28.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.27.0...v0.28.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/rabbitpubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:41:45 -08:00
dependabot[bot] ffbc6615bd build(deps): bump google.golang.org/api from 0.108.0 to 0.110.0 (#4255)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.108.0 to 0.110.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.108.0...v0.110.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-27 08:41:34 -08:00
lfhyand三千院羽 1976ca9160 add -disk to filer command (#4247)
* add -disk to filer command

* add diskType to filer.grpc

* use filer.disk when filerWebDavOptions.disk is empty

* add filer.disk to weed server command.

---------

Co-authored-by: 三千院羽 <3000y@MacBook-Pro.lan>
2023-02-25 09:48:59 -08:00
chrislu 214b7cd286 volume.fix.replication: adjust the retry checking times 2023-02-22 10:47:52 -08:00
Matt f457956948 Helm Chart Documentation and CI Updates (#4243) 2023-02-22 09:31:13 -08:00
Matt 4daa791b45 created by accident from github action (#4242) 2023-02-22 08:28:24 -08:00
LHHDZ db5515eada Fix s3api_object_list_handlers returning contents less than the specified limit when more data actually exists (#4240)
Fix when the stored data is actually enough but s3api_object_list_handlers returns less than the specified limit

Signed-off-by: changlin.shi <changlin.shi@ly.com>
2023-02-21 23:08:52 -08:00
Zachary Walters ef2f741823 Updated the deprecated ioutil dependency (#4239) 2023-02-21 19:47:33 -08:00
Chris Lu 2c4dbaedf2 Update helm_chart_release.yml 2023-02-21 11:56:29 -08:00
chrislusf 673214574a Publish seaweedfs-3.43.tgz 2023-02-21 01:05:07 +00:00
chrislu 3227e4175e 3.43 2023-02-20 16:56:05 -08:00
dependabot[bot] 439b79016d build(deps): bump wangyoucao577/go-release-action from 1.35 to 1.36 (#4226)
Bumps [wangyoucao577/go-release-action](https://github.com/wangyoucao577/go-release-action) from 1.35 to 1.36.
- [Release notes](https://github.com/wangyoucao577/go-release-action/releases)
- [Commits](https://github.com/wangyoucao577/go-release-action/compare/074de155de851f31c28201ff48135d4ecea0fb6c...efb9406897e4eff1aa2c7142223b096aec90f267)

---
updated-dependencies:
- dependency-name: wangyoucao577/go-release-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:23:59 -08:00
dependabot[bot] 508de8f492 build(deps): bump github.com/dustin/go-humanize from 1.0.0 to 1.0.1 (#4227)
Bumps [github.com/dustin/go-humanize](https://github.com/dustin/go-humanize) from 1.0.0 to 1.0.1.
- [Release notes](https://github.com/dustin/go-humanize/releases)
- [Commits](https://github.com/dustin/go-humanize/compare/v1.0.0...v1.0.1)

---
updated-dependencies:
- dependency-name: github.com/dustin/go-humanize
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:23:52 -08:00
dependabot[bot] f86c0d7751 build(deps): bump github.com/klauspost/reedsolomon from 1.11.6 to 1.11.7 (#4228)
Bumps [github.com/klauspost/reedsolomon](https://github.com/klauspost/reedsolomon) from 1.11.6 to 1.11.7.
- [Release notes](https://github.com/klauspost/reedsolomon/releases)
- [Commits](https://github.com/klauspost/reedsolomon/compare/v1.11.6...v1.11.7)

---
updated-dependencies:
- dependency-name: github.com/klauspost/reedsolomon
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:23:43 -08:00
Chris Lu d531fffc89 Revert "build(deps): bump github.com/seaweedfs/goexif from 1.0.3 to 2.0.0+incompatible" (#4232)
Revert "build(deps): bump github.com/seaweedfs/goexif from 1.0.3 to 2.0.0+incompatible (#4229)"

This reverts commit e70cf681fe.
2023-02-20 12:23:33 -08:00
dependabot[bot] e70cf681fe build(deps): bump github.com/seaweedfs/goexif from 1.0.3 to 2.0.0+incompatible (#4229)
build(deps): bump github.com/seaweedfs/goexif

Bumps [github.com/seaweedfs/goexif](https://github.com/seaweedfs/goexif) from 1.0.3 to 2.0.0+incompatible.
- [Release notes](https://github.com/seaweedfs/goexif/releases)
- [Commits](https://github.com/seaweedfs/goexif/commits)

---
updated-dependencies:
- dependency-name: github.com/seaweedfs/goexif
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:22:56 -08:00
dependabot[bot] 6b7cb33f54 build(deps): bump modernc.org/sqlite from 1.20.1 to 1.20.4 (#4230)
---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:22:49 -08:00
dependabot[bot] a554419e45 build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.42.6 to 3.42.10 (#4231)
build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3

Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.42.6 to 3.42.10.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.42.6...v3.42.10)

---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-20 12:22:40 -08:00
chrislu 28c36913b4 Revert "compile windows binary"
This reverts commit 472528f6f0.
2023-02-20 01:14:08 -08:00
raphaelauv f05261157f [DOC] readme missing compare to minio (#4223) 2023-02-19 11:34:39 -08:00
Chris Lu e3c48e987b Update helm_chart_release.yml 2023-02-18 20:40:01 -08:00
chrislu 472528f6f0 compile windows binary
fix https://github.com/seaweedfs/seaweedfs/discussions/4221
2023-02-18 13:30:12 -08:00
dependabot[bot] cbff1dbd0e build(deps): bump golang.org/x/net from 0.6.0 to 0.7.0 (#4220)
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.6.0 to 0.7.0.
- [Release notes](https://github.com/golang/net/releases)
- [Commits](https://github.com/golang/net/compare/v0.6.0...v0.7.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-17 23:21:12 -08:00
Matt 00fda86880 Added helm chart publish github actions to github pages on tagging (#4219)
* compatibility patch for csi driver

* added namespace to all component parameters

* added namespace to all component parameters

* dereference in range

* added namespace to values.yml defaults

* added namespace to s3 component

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* added helm chart to github pages

* push on all tags

* push on all tags

* push on all tags

* push on all tags

* push on all tags

* changed helm directory structure

* update charts location

* fixed dereference

* updated permissions

* updated permissions

* match current action schema
2023-02-17 22:25:50 -08:00
Matt 3f22a9db18 fix missing dereference (#4217)
fix missing dereference

fixes dereference in master statefulset
missed one, sorry
@chrislusf 

I'm working on a helm chart publish action for you that will catch these with helm lint and publish the chart to github pages.
2023-02-17 21:00:50 -08:00
Matt ca102a0b0b compatibility patch for csi driver (#4208)
* compatibility patch for csi driver

* added namespace to all component parameters

* added namespace to all component parameters

* dereference in range

* added namespace to values.yml defaults

* added namespace to s3 component
2023-02-17 17:04:43 -08:00
chrislu 9bb0a9e306 clean comments 2023-02-17 01:31:44 -08:00
chrislu 545d5d0cc3 fix for io.ReaderAt used in filer.remote.sync
fix https://github.com/seaweedfs/seaweedfs/issues/4194
2023-02-17 01:30:55 -08:00
chrislu a9aa2d581f add some more example 2023-02-16 21:30:55 -08:00
wusong 79af6b41d4 [mount] Fix read file-data in entry.content (#4207) 2023-02-15 07:18:56 -08:00
dependabot[bot] 896c6b8d1a build(deps): bump github.com/fclairamb/ftpserverlib from 0.20.0 to 0.21.0 (#4203)
build(deps): bump github.com/fclairamb/ftpserverlib

Bumps [github.com/fclairamb/ftpserverlib](https://github.com/fclairamb/ftpserverlib) from 0.20.0 to 0.21.0.
- [Release notes](https://github.com/fclairamb/ftpserverlib/releases)
- [Commits](https://github.com/fclairamb/ftpserverlib/compare/v0.20.0...v0.21.0)

---
updated-dependencies:
- dependency-name: github.com/fclairamb/ftpserverlib
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-13 09:35:02 -08:00
dependabot[bot] 3d7baa30cd build(deps): bump github.com/go-redsync/redsync/v4 from 4.7.1 to 4.8.1 (#4202)
Bumps [github.com/go-redsync/redsync/v4](https://github.com/go-redsync/redsync) from 4.7.1 to 4.8.1.
- [Release notes](https://github.com/go-redsync/redsync/releases)
- [Commits](https://github.com/go-redsync/redsync/compare/v4.7.1...v4.8.1)

---
updated-dependencies:
- dependency-name: github.com/go-redsync/redsync/v4
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-13 09:34:51 -08:00
dependabot[bot] 1134de8cbe build(deps): bump github.com/rabbitmq/amqp091-go from 1.5.0 to 1.7.0 (#4200)
Bumps [github.com/rabbitmq/amqp091-go](https://github.com/rabbitmq/amqp091-go) from 1.5.0 to 1.7.0.
- [Release notes](https://github.com/rabbitmq/amqp091-go/releases)
- [Changelog](https://github.com/rabbitmq/amqp091-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/rabbitmq/amqp091-go/compare/v1.5.0...v1.7.0)

---
updated-dependencies:
- dependency-name: github.com/rabbitmq/amqp091-go
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-13 09:34:38 -08:00
dependabot[bot] 292e98cc22 build(deps): bump golang.org/x/tools from 0.5.0 to 0.6.0 (#4199)
Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.5.0 to 0.6.0.
- [Release notes](https://github.com/golang/tools/releases)
- [Commits](https://github.com/golang/tools/compare/v0.5.0...v0.6.0)

---
updated-dependencies:
- dependency-name: golang.org/x/tools
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-13 09:34:27 -08:00
chrislu 5869945f16 avoid infinite loop
fix https://github.com/seaweedfs/seaweedfs/issues/4195#issuecomment-1426100904
2023-02-10 13:05:04 -08:00
chrislu e037c71ec3 adjust text 2023-02-10 13:04:29 -08:00
chrislu 67b8c2853a add line return 2023-02-10 12:53:43 -08:00
chrislu 302155a6fa fix compilation 2023-02-10 12:32:59 -08:00
chrislu 5520b81551 add some error messages for args
related to https://github.com/seaweedfs/seaweedfs/issues/4115
2023-02-10 12:30:53 -08:00
91 changed files with 1003 additions and 577 deletions

No files matched your search

+4 -4
View File
@@ -44,7 +44,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -93,7 +93,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
with:
buildkitd-flags: "--debug"
-
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
with:
buildkitd-flags: "--debug"
-
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+22
View File
@@ -0,0 +1,22 @@
name: "helm: publish charts"
on:
push:
tags:
- '*'
permissions:
contents: write
pages: write
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@master
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
+51
View File
@@ -0,0 +1,51 @@
name: "helm: lint and test charts"
on:
push:
branches: [ master ]
paths: ['k8s/**']
pull_request:
branches: [ master ]
paths: ['k8s/**']
permissions:
contents: read
jobs:
lint-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.10.0
- uses: actions/setup-python@v4
with:
python-version: '3.9'
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.1
- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --chart-dirs k8s/charts)
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
- name: Run chart-testing (lint)
run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
- name: Create kind cluster
uses: helm/kind-action@v1.5.0
- name: Run chart-testing (install)
run: ct install --all --chart-dirs k8s/charts
+3 -2
View File
@@ -8,7 +8,7 @@
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/seaweedfs)](https://artifacthub.io/packages/search?repo=seaweedfs)
![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png)
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
-->
### Gold Sponsors
[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
---
@@ -66,6 +66,7 @@ Table of Contents
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
* [Compared to GlusterFS](#compared-to-glusterfs)
* [Compared to Ceph](#compared-to-ceph)
* [Compared to Minio](#compared-to-minio)
* [Dev Plan](#dev-plan)
* [Installation Guide](#installation-guide)
* [Disk Related Topics](#disk-related-topics)
+2 -1
View File
@@ -62,7 +62,8 @@ k8s: build
dev_registry: build
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate: build
dev_replicate:
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
+56 -55
View File
@@ -3,22 +3,23 @@ module github.com/seaweedfs/seaweedfs
go 1.20
require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/pubsub v1.28.0
cloud.google.com/go/storage v1.29.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.44.189
github.com/aws/aws-sdk-go v1.44.219
github.com/cenkalti/backoff/v4 v4.2.0
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.0
github.com/dustin/go-humanize v1.0.1
github.com/eapache/go-resiliency v1.3.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
github.com/eapache/queue v1.1.0 // indirect
@@ -27,11 +28,11 @@ require (
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.20.0
github.com/fclairamb/ftpserverlib v0.21.0
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.7.1
github.com/go-redsync/redsync/v4 v4.8.1
github.com/go-sql-driver/mysql v1.7.0
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
@@ -58,15 +59,15 @@ require (
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/klauspost/reedsolomon v1.11.6
github.com/klauspost/reedsolomon v1.11.7
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.7
github.com/linxGnu/grocksdb v1.7.14
github.com/linxGnu/grocksdb v1.7.15
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -78,10 +79,10 @@ require (
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/common v0.39.0 // indirect
github.com/prometheus/procfs v0.9.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.9.0 // indirect
@@ -108,74 +109,75 @@ require (
go.etcd.io/etcd/client/v3 v3.5.7
go.mongodb.org/mongo-driver v1.11.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.28.0
gocloud.dev v0.29.0
gocloud.dev/pubsub/natspubsub v0.28.0
gocloud.dev/pubsub/rabbitpubsub v0.27.0
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.5.0
golang.org/x/oauth2 v0.2.0 // indirect
golang.org/x/sys v0.4.0
golang.org/x/text v0.6.0 // indirect
golang.org/x/tools v0.5.0
gocloud.dev/pubsub/rabbitpubsub v0.29.0
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
golang.org/x/image v0.6.0
golang.org/x/net v0.8.0
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.6.0
golang.org/x/text v0.8.0 // indirect
golang.org/x/tools v0.6.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.108.0
google.golang.org/api v0.112.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.52.0
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
google.golang.org/grpc v1.53.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.2 // indirect
modernc.org/libc v1.22.3 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.20.1
modernc.org/sqlite v1.21.0
modernc.org/strutil v1.1.3
modernc.org/token v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.4.0
github.com/arangodb/go-driver v1.4.1
github.com/arangodb/go-driver v1.5.2
github.com/armon/go-metrics v0.4.1
github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
github.com/hanwen/go-fuse/v2 v2.2.0
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/rabbitmq/amqp091-go v1.5.0
github.com/rabbitmq/amqp091-go v1.7.0
github.com/schollz/progressbar/v3 v3.13.0
github.com/tikv/client-go/v2 v2.0.5
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
require (
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
github.com/aws/smithy-go v1.13.4 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.0
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
@@ -183,18 +185,18 @@ require (
github.com/fclairamb/go-log v0.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
@@ -212,7 +214,6 @@ require (
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
@@ -229,11 +230,11 @@ require (
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.4.0 // indirect
golang.org/x/term v0.6.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
+319 -182
View File
File diff suppressed because it is too large. Load diff
+16
View File
@@ -0,0 +1,16 @@
# Artifact Hub repository metadata file
#
# Some settings like the verified publisher flag or the ignored packages won't
# be applied until the next time the repository is processed. Please keep in
# mind that the repository won't be processed if it has not changed since the
# last time it was processed. Depending on the repository kind, this is checked
# in a different way. For Helm http based repositories, we consider it has
# changed if the `index.yaml` file changes. For git based repositories, it does
# when the hash of the last commit in the branch you set up changes. This does
# NOT apply to ownership claim operations, which are processed immediately.
#
repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
# owners: # (optional, used to claim repository ownership)
# - name: username
# email: email
File renamed without changes.
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.42"
version: "3.42"
appVersion: "3.44"
version: "3.44"
+81
View File
@@ -0,0 +1,81 @@
# SEAWEEDFS - helm chart (2.x+)
## Getting Started
### Add the helm repo
`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
### Install the helm chart
`helm install seaweedfs seaweedfs/seaweedfs`
### (Recommended) Provide `values.yaml`
`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
## Info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
## Prerequisites
### Database
leveldb is the default database this only supports one filer replica.
To have multiple filers a external datastore is recommened.
Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
```
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
### Node Labels
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
Here is an example:
* s3/filer/master needs the label **sw-backend=true**
* volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s:
```
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
```
on production k8s deployment you will want each pod to have a different host,
especially the volume server and the masters, all pods (master/volume/filer)
should have anti-affinity rules to disallow running multiple component pods on the same host.
If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
```affinity: ""```
## PVC - storage class ###
On the volume stateful set added support for k8s PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class
for your deployment.
## current instances config (AIO):
1 instance for each type (master/filer+s3/volume)
You can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding labels if applicable.
Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
@@ -97,6 +97,12 @@ spec:
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
valueFrom: {{ toYaml $value | nindent 16 }}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -162,7 +168,7 @@ spec:
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
{{- end }}
{{- end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name: seaweedfs-filer-log-volume
mountPath: "/logs/"
@@ -133,8 +133,8 @@ spec:
{{- if .Values.master.garbageThreshold }}
-garbageThreshold={{ .Values.master.garbageThreshold }} \
{{- end }}
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name : data-{{ .Release.Namespace }}
mountPath: /data
@@ -105,7 +105,7 @@ spec:
{{- if .Values.s3.auditLogConfig }}
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
{{- end }}
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
-filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
volumeMounts:
- name: logs
mountPath: "/logs/"
@@ -138,9 +138,9 @@ spec:
-fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
{{- end }}
-minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
-compactionMBps={{ .Values.volume.compactionMBps }} \
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name: data
mountPath: "{{ .Values.volume.dir }}/"
@@ -187,7 +187,7 @@ spec:
port: {{ .Values.volume.port }}
scheme: HTTP
initialDelaySeconds: 15
periodSeconds: 90
periodSeconds: 15
successThreshold: 1
failureThreshold: 100
timeoutSeconds: 30
@@ -5,7 +5,7 @@ global:
repository: ""
imageName: chrislusf/seaweedfs
imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret
imagePullSecrets: ""
restartPolicy: Always
loggingLevel: 1
enableSecurity: false
@@ -22,8 +22,8 @@ global:
replicationPlacment: "001"
extraEnvironmentVars:
WEED_CLUSTER_DEFAULT: "sw"
WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
image:
registry: ""
@@ -44,11 +44,11 @@ master:
volumePreallocate: false
volumeSizeLimitMB: 1000
loggingOverrideLevel: null
#number of seconds between heartbeats, default 5
# number of seconds between heartbeats, default 5
pulseSeconds: null
#threshold to vacuum and reclaim spaces, default 0.3 (30%)
# threshold to vacuum and reclaim spaces, default 0.3 (30%)
garbageThreshold: null
#Prometheus push interval in seconds, default 15
# Prometheus push interval in seconds, default 15
metricsIntervalSec: 15
# replication type is XYZ:
# X number of replica in other data centers
@@ -112,10 +112,10 @@ master:
# nodeSelector labels for master pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
# nodeSelector: |
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# used to assign priority to master pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -241,7 +241,7 @@ volume:
# should map directly to the value of the resources field for a PodSpec,
# formatted as a multi-line string. By default no direct resource request
# is made.
resources: null
resources: {}
# Toleration Settings for server pods
# This should be a multi-line string matching the Toleration array
@@ -251,10 +251,10 @@ volume:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
# nodeSelector: |
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-volume: "true"
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-volume: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -358,10 +358,10 @@ filer:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
# nodeSelector: |
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -389,7 +389,7 @@ filer:
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
extraEnvironmentVars:
WEED_MYSQL_ENABLED: "true"
WEED_MYSQL_ENABLED: "false"
WEED_MYSQL_HOSTNAME: "mysql-db-host"
WEED_MYSQL_PORT: "3306"
WEED_MYSQL_DATABASE: "sw_database"
@@ -400,17 +400,29 @@ filer:
# enable usage of memsql as filer backend
WEED_MYSQL_INTERPOLATEPARAMS: "true"
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
WEED_LEVELDB2_ENABLED: "false"
WEED_LEVELDB2_ENABLED: "true"
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
# directories under this folder will be automatically creating a separate bucket
WEED_FILER_BUCKETS_FOLDER: "/buckets"
# secret env variables
secretExtraEnvironmentVars: []
# WEED_POSTGRES_USERNAME:
# secretKeyRef:
# name: postgres-credentials
# key: username
# WEED_POSTGRES_PASSWORD:
# secretKeyRef:
# name: postgres-credentials
# key: password
s3:
enabled: true
port: 8333
#allow empty folders
# allow empty folders
allowEmptyFolder: false
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
@@ -430,7 +442,7 @@ s3:
port: 8333
metricsPort: 9327
loggingOverrideLevel: null
#allow empty folders
# allow empty folders
allowEmptyFolder: true
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
@@ -449,7 +461,7 @@ s3:
# should map directly to the value of the resources field for a PodSpec,
# formatted as a multi-line string. By default no direct resource request
# is made.
resources: null
resources: {}
# Toleration Settings for server pods
# This should be a multi-line string matching the Toleration array
@@ -459,10 +471,10 @@ s3:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
# nodeSelector: |
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -479,5 +491,5 @@ certificates:
ipAddresses: []
keyAlgorithm: rsa
keySize: 2048
duration: 2160h # 90d
renewBefore: 360h # 15d
duration: 2160h # 90d
renewBefore: 360h # 15d
-62
View File
@@ -1,62 +0,0 @@
# SEAWEEDFS - helm chart (2.x)
## Info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
with ENV.
* cert config exists and can be enabled, but not been tested.
## Prerequisites
### Database
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
```
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
### Node Labels
Kubernetes node have labels which help to define which node(Host) will run which pod:
* s3/filer/master needs the label **sw-backend=true**
* volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s:
```
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
```
on production k8s deployment you will want each pod to have a different host,
especially the volume server & the masters, currently all pods (master/volume/filer)
have anti-affinity rule to disallow running multiple pod type on the same host.
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
please set/update the corresponding affinity rule in values.yaml to an empty one:
```affinity: ""```
## PVC - storage class ###
on the volume stateful set added support for K8S PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class
for your deployment.
## current instances config (AIO):
1 instance for each type (master/filer+s3/volume)
you can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding labels.
most of the configuration are available through values.yaml
+4
View File
@@ -5,6 +5,10 @@
<artifactId>copier</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>18</maven.compiler.source>
<maven.compiler.target>18</maven.compiler.target>
</properties>
<name>copier</name>
<url>http://maven.apache.org</url>
@@ -10,6 +10,8 @@ import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
@@ -42,6 +44,19 @@ public class HighLevelMultipartUpload {
.withS3Client(s3Client)
.build();
if (!s3Client.doesBucketExistV2(bucketName)) {
// Because the CreateBucketRequest object doesn't specify a region, the
// bucket is created in the region specified in the client.
s3Client.createBucket(new CreateBucketRequest(bucketName));
// Verify that the bucket was created by retrieving it and checking its location.
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
System.out.println("Bucket location: " + bucketLocation);
} else {
System.out.println("Bucket already exists");
}
// TransferManager processes all transfers asynchronously,
// so this call returns immediately.
Upload upload = tm.upload(bucketName, keyName, file);
+8
View File
@@ -60,6 +60,7 @@ type FilerOptions struct {
localSocket *string
showUIDirectoryDelete *bool
downloadMaxMBps *int
diskType *string
}
func init() {
@@ -89,6 +90,7 @@ func init() {
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
// start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -183,6 +185,11 @@ func runFiler(cmd *Command, args []string) bool {
if *filerStartWebDav {
filerWebDavOptions.filer = &filerAddress
if *filerWebDavOptions.disk == "" {
filerWebDavOptions.disk = f.diskType
}
go func(delay time.Duration) {
time.Sleep(delay * time.Second)
filerWebDavOptions.startWebDav()
@@ -243,6 +250,7 @@ func (fo *FilerOptions) startFiler() {
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
DiskType: *fo.diskType,
})
if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err)
+7 -6
View File
@@ -2,13 +2,14 @@ package command
import (
"fmt"
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
"strings"
"time"
hashicorpRaft "github.com/hashicorp/raft"
"golang.org/x/exp/slices"
"github.com/gorilla/mux"
@@ -163,11 +164,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
RaftResumeState: *masterOption.raftResumeState,
HeartbeatInterval: *masterOption.heartbeatInterval,
ElectionTimeout: *masterOption.electionTimeout,
RaftBootstrap: *m.raftBootstrap,
RaftBootstrap: *masterOption.raftBootstrap,
}
var raftServer *weed_server.RaftServer
var err error
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
glog.Fatalf("NewHashicorpRaftServer: %s", err)
}
@@ -180,7 +181,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
}
// starting grpc server
@@ -191,7 +192,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
raftServer.TransportManager.Register(grpcS)
} else {
protobuf.RegisterRaftServer(grpcS, raftServer)
@@ -204,7 +205,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go grpcS.Serve(grpcL)
timeSleep := 1500 * time.Millisecond
if !*m.raftHashicorp {
if !*masterOption.raftHashicorp {
go func() {
time.Sleep(timeSleep)
+5
View File
@@ -97,6 +97,7 @@ func init() {
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
@@ -115,6 +116,7 @@ func init() {
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
@@ -253,6 +255,9 @@ func runServer(cmd *Command, args []string) bool {
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
if *isStartingFiler {
if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
filerOptions.diskType = serverOptions.v.diskType
}
go func() {
time.Sleep(1 * time.Second)
filerOptions.startFiler()
+3 -4
View File
@@ -11,7 +11,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -215,7 +214,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
buf, err := ioutil.ReadAll(res.Body)
buf, err := io.ReadAll(res.Body)
if err != nil {
return Release{}, err
}
@@ -265,7 +264,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
buf, err := ioutil.ReadAll(res.Body)
buf, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
@@ -337,7 +336,7 @@ func extractToFile(buf []byte, filename, target string) error {
// Write everything to a temp file
dir := filepath.Dir(target)
new, err := ioutil.TempFile(dir, "weed")
new, err := os.CreateTemp(dir, "weed")
if err != nil {
return err
}
+15 -22
View File
@@ -300,31 +300,24 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
return nil
}
if c.chunkView == nil {
return io.EOF
}
// glog.V(2).Infof("c.chunkView: %v buffer:[%d,%d) offset:%d totalSize:%d", c.chunkView, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)), offset, c.totalSize)
// fmt.Printf("fetch for offset %d\n", offset)
c.chunkView = c.chunkView.Next
if c.chunkView == nil {
return io.EOF
}
// positioning within the new chunk
chunk := c.chunkView.Value
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
return c.fetchChunkToBuffer(chunk)
}
} else {
for p := c.head; p != nil; p = p.Next {
chunk = p.Value
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
return c.fetchChunkToBuffer(chunk)
}
// find a possible chunk view
p := c.chunkView
for p != nil {
chunk := p.Value
// glog.V(2).Infof("prepareBufferFor check chunk:[%d,%d)", chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
c.chunkView = p
return c.fetchChunkToBuffer(chunk)
}
}
if offset < c.bufferOffset {
p = p.Prev
} else {
p = p.Next
}
}
return io.EOF
+47 -47
View File
@@ -1,47 +1,47 @@
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}
+1 -1
View File
@@ -53,7 +53,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
return 0, 0, io.EOF
}
if offset+int64(len(buff)) <= int64(len(entry.Content)) {
if offset < int64(len(entry.Content)) {
totalRead := copy(buff, entry.Content[offset:])
glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
return int64(totalRead), 0, nil
+10 -7
View File
@@ -3,14 +3,16 @@ package mount
import (
"context"
"fmt"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"strings"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
/** Create a directory
@@ -54,9 +56,10 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
defer wfs.mapPbIdFromFilerToLocal(newEntry)
request := &filer_pb.CreateEntryRequest{
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
glog.V(1).Infof("mkdir: %v", request)
+6 -3
View File
@@ -2,12 +2,14 @@ package mount
import (
"context"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"syscall"
"time"
)
/*
@@ -72,7 +74,8 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
HardLinkId: oldEntry.HardLinkId,
HardLinkCounter: oldEntry.HardLinkCounter,
},
Signatures: []int32{wfs.signature},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
// apply changes to the filer, and also apply to local metaCache
+8 -5
View File
@@ -3,13 +3,15 @@ package mount
import (
"context"
"fmt"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
/** Create a symbolic link */
@@ -42,7 +44,8 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
SymlinkTarget: target,
},
},
Signatures: []int32{wfs.signature},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+1 -1
View File
@@ -1,6 +1,6 @@
package s3_constants
//Amazon S3 predefined groups
// Amazon S3 predefined groups
var (
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+47 -39
View File
@@ -146,57 +146,65 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
// check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
for {
empty := true
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
empty = false
if entry.IsDirectory {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: "STANDARD",
})
cursor.maxKeys--
}
} else {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: "STANDARD",
StorageClass: StorageClass(storageClass),
})
cursor.maxKeys--
}
} else {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: StorageClass(storageClass),
})
cursor.maxKeys--
})
if doErr != nil {
return doErr
}
})
if doErr != nil {
return doErr
}
if !cursor.isTruncated {
nextMarker = ""
} else {
if requestDir != "" {
nextMarker = requestDir + "/" + nextMarker
if cursor.isTruncated {
if requestDir != "" {
nextMarker = requestDir + "/" + nextMarker
}
break
} else if empty {
nextMarker = ""
break
} else {
// start next loop
marker = nextMarker
}
}
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
"google.golang.org/grpc/security/advancedtls"
"io/ioutil"
"os"
"strings"
"time"
@@ -136,7 +136,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
}
func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
clientCerts, err := ioutil.ReadFile(clientCertFile)
clientCerts, err := os.ReadFile(clientCertFile)
if err != nil {
glog.Fatal(err)
}
+4
View File
@@ -295,6 +295,10 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
if req.DiskType == "" {
req.DiskType = fs.option.DiskType
}
so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack, req.DataNode)
if err != nil {
glog.V(3).Infof("AssignVolume: %v", err)
+2 -1
View File
@@ -154,10 +154,11 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
Offset: localOffset,
Size: uint64(size),
ModifiedTsNs: time.Now().Unix(),
ModifiedTsNs: time.Now().UnixNano(),
ETag: etag,
Fid: &filer_pb.FileId{
VolumeId: uint32(fileId.VolumeId),
+1 -3
View File
@@ -69,6 +69,7 @@ type FilerOption struct {
ConcurrentUploadLimit int64
ShowUIDirectoryDelete bool
DownloadMaxBytesPs int64
DiskType string
}
type FilerServer struct {
@@ -194,9 +195,6 @@ func (fs *FilerServer) checkWithMaster() {
return fmt.Errorf("get master %s configuration: %v", master, err)
}
fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
if fs.option.DefaultReplication == "" {
fs.option.DefaultReplication = resp.DefaultReplication
}
return nil
})
if readErr == nil {
@@ -27,7 +27,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
limit, limit_err := strconv.Atoi(r.FormValue("limit"))
if limit_err != nil {
limit = 100
limit = fs.option.DirListingLimit
}
lastFileName := r.FormValue("lastFileName")
+7 -1
View File
@@ -4,12 +4,13 @@ import (
"context"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"net/http"
"os"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -97,6 +98,11 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
return
}
// When DiskType is empty,use filer's -disk
if so.DiskType == "" {
so.DiskType = fs.option.DiskType
}
if query.Has("mv.from") {
fs.move(ctx, w, r, so)
} else {
@@ -49,7 +49,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
writeJsonError(w, r, 499, err)
} else if strings.HasSuffix(err.Error(), "is a file") {
} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
writeJsonError(w, r, http.StatusConflict, err)
} else {
writeJsonError(w, r, http.StatusInternalServerError, err)
+9 -2
View File
@@ -309,12 +309,19 @@ func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.V
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(req.ReplicaPlacement))
vl := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, needle.LoadTTLFromUint32(req.Ttl), types.ToDiskType(req.DiskType))
dataNodes := ms.Topo.Lookup(req.Collection, needle.VolumeId(req.VolumeId))
if req.IsReadonly {
for _, dn := range dataNodes {
if dn.Ip == req.Ip && dn.Port == int(req.Port) {
}
}
}
for _, dn := range dataNodes {
if dn.Ip == req.Ip && dn.Port == int(req.Port) {
if req.IsReadonly {
vl.SetVolumeUnavailable(dn, needle.VolumeId(req.VolumeId))
vl.SetVolumeReadOnly(dn, needle.VolumeId(req.VolumeId))
} else {
vl.SetVolumeAvailable(dn, needle.VolumeId(req.VolumeId), false)
vl.SetVolumeWritable(dn, needle.VolumeId(req.VolumeId))
}
}
}
+1 -2
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
transport "github.com/Jille/raft-grpc-transport"
"io"
"io/ioutil"
"math/rand"
"os"
"path"
@@ -90,7 +89,7 @@ func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
}
func (s *StateMachine) Restore(r io.ReadCloser) error {
b, err := ioutil.ReadAll(r)
b, err := io.ReadAll(r)
if err != nil {
return err
}
+65 -20
View File
@@ -12,9 +12,11 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/util"
"golang.org/x/exp/slices"
"io"
"math"
"strings"
"sync"
"time"
)
@@ -24,10 +26,14 @@ func init() {
type commandFsVerify struct {
env *CommandEnv
volumeServers []pb.ServerAddress
volumeIds map[uint32][]pb.ServerAddress
verbose *bool
concurrency *int
modifyTimeAgoAtSec int64
writer io.Writer
waitChan map[string]chan struct{}
waitChanLock sync.RWMutex
}
func (c *commandFsVerify) Name() string {
@@ -45,10 +51,10 @@ func (c *commandFsVerify) Help() string {
func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
c.env = commandEnv
c.writer = writer
fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server")
if err = fsVerifyCommand.Parse(args); err != nil {
return err
@@ -60,13 +66,29 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
}
c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
c.volumeIds = make(map[uint32][]pb.ServerAddress)
c.waitChan = make(map[string]chan struct{})
c.volumeServers = []pb.ServerAddress{}
defer func() {
c.modifyTimeAgoAtSec = 0
c.volumeIds = nil
c.waitChan = nil
c.volumeServers = nil
}()
if err := c.collectVolumeIds(); err != nil {
return parseErr
}
fCount, eConut, terr := c.verifyTraverseBfs(path)
if *c.concurrency > 0 {
for _, volumeServer := range c.volumeServers {
volumeServerStr := string(volumeServer)
c.waitChan[volumeServerStr] = make(chan struct{}, *c.concurrency)
defer close(c.waitChan[volumeServerStr])
}
}
fCount, eConut, terr := c.verifyTraverseBfs(path)
if terr == nil {
fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut)
}
@@ -76,7 +98,6 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
}
func (c *commandFsVerify) collectVolumeIds() error {
c.volumeIds = make(map[uint32][]pb.ServerAddress)
topologyInfo, _, err := collectTopologyInfo(c.env, 0)
if err != nil {
return err
@@ -84,15 +105,19 @@ func (c *commandFsVerify) collectVolumeIds() error {
eachDataNode(topologyInfo, func(dc string, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
for _, diskInfo := range nodeInfo.DiskInfos {
for _, vi := range diskInfo.VolumeInfos {
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], pb.NewServerAddressFromDataNode(nodeInfo))
volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
if !slices.Contains(c.volumeServers, volumeServer) {
c.volumeServers = append(c.volumeServers, volumeServer)
}
}
}
})
return nil
}
func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.ServerAddress) error {
err := operation.WithVolumeServerClient(false, *volumeServer, c.env.option.GrpcDialOption,
func (c *commandFsVerify) verifyEntry(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error {
err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeNeedleStatus(context.Background(),
&volume_server_pb.VolumeNeedleStatusRequest{
@@ -104,9 +129,6 @@ func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.
if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
return err
}
if *c.verbose {
fmt.Fprintf(c.writer, ".")
}
return nil
}
@@ -140,27 +162,50 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
func(outputChan chan interface{}) {
for itemEntry := range outputChan {
i := itemEntry.(*ItemEntry)
fileMsg := fmt.Sprintf("file:%s needle status ", i.path)
if *c.verbose {
fmt.Fprintf(c.writer, fileMsg)
fileMsg = ""
}
itemPath := string(i.path)
fileMsg := fmt.Sprintf("file:%s", itemPath)
errItem := make(map[string]error)
errItemLock := sync.RWMutex{}
for _, chunk := range i.chunks {
if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
for _, volumeServer := range volumeIds {
if err = c.verifyEntry(chunk.Fid, &volumeServer); err != nil {
fmt.Fprintf(c.writer, "%sfailed verify %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
break
if *c.concurrency == 0 {
if err = c.verifyEntry(volumeServer, chunk.Fid); err != nil {
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
}
continue
}
c.waitChanLock.RLock()
waitChan, ok := c.waitChan[string(volumeServer)]
c.waitChanLock.RUnlock()
if !ok {
fmt.Fprintf(c.writer, "%s failed to get channel for %s chunk: %d:%d: %+v\n",
string(volumeServer), fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
continue
}
waitChan <- struct{}{}
go func(fId *filer_pb.FileId, path string, volumeServer pb.ServerAddress, msg string) {
if err = c.verifyEntry(volumeServer, fId); err != nil {
errItemLock.Lock()
errItem[path] = err
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
msg, fId.VolumeId, fId.FileKey, err)
errItemLock.Unlock()
}
<-waitChan
}(chunk.Fid, itemPath, volumeServer, fileMsg)
}
} else {
err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
fmt.Fprintf(c.writer, "%sfailed verify chunk %d:%d: %+v\n",
fmt.Fprintf(c.writer, "%s %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
break
}
}
errItemLock.RLock()
err, _ = errItem[itemPath]
errItemLock.RUnlock()
if err != nil {
errCount++
@@ -168,7 +213,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
}
if *c.verbose {
fmt.Fprintf(c.writer, " verifed\n")
fmt.Fprintf(c.writer, "%s needles:%d verifed\n", fileMsg, len(i.chunks))
}
fileCount++
}
+1 -1
View File
@@ -63,7 +63,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
collectionSize = collectionInfo.Size
fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
}
fmt.Fprintf(writer, " %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
fmt.Fprintf(writer, " %s\tsize:%.0f\tchunk:%.0f", entry.Name, collectionSize, fileCount)
if entry.Quota > 0 {
fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
}
+1 -1
View File
@@ -330,7 +330,7 @@ func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*Vol
}
if candidateVolume.RemoteStorageName != "" {
return false, nil
return false, fmt.Errorf("does not move volume in remove storage")
}
if candidateVolume.ReplicaPlacement > 0 {
+1 -1
View File
@@ -58,7 +58,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'")
skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes")
noDelete := volFixReplicationCommand.Bool("noDelete", false, "Do not delete over-replicated volumes, only fix under-replication")
retryCount := volFixReplicationCommand.Int("retry", 0, "how many times to retry")
retryCount := volFixReplicationCommand.Int("retry", 5, "how many times to retry")
volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle")
if err = volFixReplicationCommand.Parse(args); err != nil {
+5 -5
View File
@@ -20,7 +20,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
@@ -164,6 +163,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
if *c.findMissingChunksInFiler {
// collect all filer file ids and paths
if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectCutoffFromAtNs); err != nil {
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
}
@@ -409,7 +409,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
}
buf.Write(resp.FileContent)
}
if vinfo.isReadOnly == false {
if !vinfo.isReadOnly {
index, err := idx.FirstInvalidIndex(buf.Bytes(),
func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
@@ -419,12 +419,12 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
Size: int32(size),
})
if err != nil {
return false, fmt.Errorf("to read needle meta with id %d from volume %d with error %v", key, volumeId, err)
return false, fmt.Errorf("read needle meta with id %d from volume %d: %v", key, volumeId, err)
}
return resp.AppendAtNs <= cutoffFrom, nil
})
if err != nil {
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v\n", volumeId, err)
} else {
buf.Truncate(index * types.NeedleMapEntrySize)
}
@@ -535,7 +535,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
_, err = io.ReadAll(resp.Body)
if err != nil {
fmt.Fprintf(c.writer, "DELETE response error: %v\n", err)
}
+15 -17
View File
@@ -19,8 +19,8 @@ func init() {
type commandVolumeServerEvacuate struct {
topologyInfo *master_pb.TopologyInfo
targetServer string
volumeRack string
targetServer *string
volumeRack *string
}
func (c *commandVolumeServerEvacuate) Name() string {
@@ -49,8 +49,8 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
volumeRack := vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
targetServer := vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
c.volumeRack = vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
c.targetServer = vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
retryCount := vsEvacuateCommand.Int("retry", 0, "how many times to retry")
@@ -63,15 +63,10 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
return
}
if *volumeServer == "" && *volumeRack == "" {
if *volumeServer == "" && *c.volumeRack == "" {
return fmt.Errorf("need to specify volume server by -node=<host>:<port> or source rack")
}
if *targetServer != "" {
c.targetServer = *targetServer
}
if *volumeRack != "" {
c.volumeRack = *volumeRack
}
for i := 0; i < *retryCount+1; i++ {
if err = c.volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer); err == nil {
return nil
@@ -93,6 +88,9 @@ func (c *commandVolumeServerEvacuate) volumeServerEvacuate(commandEnv *CommandEn
if err != nil {
return err
}
defer func() {
c.topologyInfo = nil
}()
if err := c.evacuateNormalVolumes(commandEnv, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
return err
@@ -237,14 +235,14 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][
func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, thisServer string) (thisNodes []*Node, otherNodes []*Node) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (c.volumeRack != "" && node.rack == c.volumeRack) {
if node.info.Id == thisServer || (*c.volumeRack != "" && node.rack == *c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if c.volumeRack != "" && c.volumeRack == node.rack {
if *c.volumeRack != "" && *c.volumeRack == node.rack {
continue
}
if c.targetServer != "" && c.targetServer != node.info.Id {
if *c.targetServer != "" && *c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)
@@ -254,14 +252,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this
func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (c.volumeRack != "" && string(node.rack) == c.volumeRack) {
if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if c.volumeRack != "" && c.volumeRack == string(node.rack) {
if *c.volumeRack != "" && *c.volumeRack == string(node.rack) {
continue
}
if c.targetServer != "" && c.targetServer != node.info.Id {
if *c.targetServer != "" && *c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)
@@ -6,7 +6,12 @@ import (
)
func TestVolumeServerEvacuate(t *testing.T) {
c := commandVolumeServerEvacuate{}
targetServer := ""
volumeRack := "rack1"
c := commandVolumeServerEvacuate{
targetServer: &targetServer,
volumeRack: &volumeRack,
}
c.topologyInfo = parseOutput(topoData)
volumeServer := "192.168.1.4:8080"
+7
View File
@@ -3,6 +3,7 @@ package shell
import (
"context"
"flag"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"io"
@@ -41,6 +42,12 @@ func (c *commandVolumeUnmount) Do(args []string, commandEnv *CommandEnv, writer
if err = volUnmountCommand.Parse(args); err != nil {
return nil
}
if *nodeStr == "" {
return fmt.Errorf("-node option is required")
}
if *volumeIdInt == 0 {
return fmt.Errorf("-volumeId option is required")
}
if err = commandEnv.confirmIsLocked(args); err != nil {
return
+10 -2
View File
@@ -1,7 +1,6 @@
package stats
import (
"fmt"
"log"
"net"
"net/http"
@@ -282,12 +281,21 @@ func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
}
}
func JoinHostPort(host string, port int) string {
portStr := strconv.Itoa(port)
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
return host + ":" + portStr
}
return net.JoinHostPort(host, portStr)
}
func StartMetricsServer(ip string, port int) {
if port == 0 {
return
}
http.Handle("/metrics", promhttp.HandlerFor(Gather, promhttp.HandlerOpts{}))
log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil))
log.Fatal(http.ListenAndServe(JoinHostPort(ip, port), nil))
}
func SourceName(port uint32) string {
+5 -1
View File
@@ -66,8 +66,12 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
if err != nil {
return err
}
var index int
index, err = n.readNeedleDataVersion2NonData(metaSlice)
if size.IsValid() {
index, err = n.readNeedleDataVersion2NonData(metaSlice)
}
n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize]))
if version == Version3 {
n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize])
+1 -1
View File
@@ -16,7 +16,7 @@ func TestFastLoadingNeedleMapMetrics(t *testing.T) {
for i := 0; i < 10000; i++ {
nm.Put(Uint64ToNeedleId(uint64(i+1)), Uint32ToOffset(uint32(0)), Size(1))
if rand.Float32() < 0.2 {
if rand.Float32() < 0.2 && i > 0 {
nm.Delete(Uint64ToNeedleId(uint64(rand.Int63n(int64(i))+1)), Uint32ToOffset(uint32(0)))
}
}
+6 -2
View File
@@ -41,7 +41,9 @@ func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
expectedLastUpdateTime += 2000
}
}
@@ -86,7 +88,9 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
expectedLastUpdateTime += 2000
}
}
+6 -5
View File
@@ -2,15 +2,16 @@ package topology
import (
"errors"
"math/rand"
"strings"
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"math/rand"
"strings"
"sync"
"sync/atomic"
)
type NodeId string
@@ -73,7 +74,7 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption
n.RUnlock()
if len(candidates) < numberOfNodes {
glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
return nil, nil, errors.New("No enough data node found!")
return nil, nil, errors.New("Not enough data nodes found!")
}
//pick nodes randomly by weights, the node picked earlier has higher final weights
+19 -13
View File
@@ -11,6 +11,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
backoff "github.com/cenkalti/backoff/v4"
hashicorpRaft "github.com/hashicorp/raft"
"github.com/seaweedfs/raft"
@@ -35,7 +37,7 @@ type Topology struct {
volumeSizeLimit uint64
replicationAsMin bool
isDisableVacuum bool
isDisableVacuum bool
Sequence sequence.Sequencer
@@ -96,19 +98,23 @@ func (t *Topology) IsLeader() bool {
}
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
for count := 0; count < 3; count++ {
l, err = t.MaybeLeader()
if err != nil {
return
}
if l != "" {
break
}
time.Sleep(time.Duration(5+count) * time.Second)
exponentialBackoff := backoff.NewExponentialBackOff()
exponentialBackoff.InitialInterval = 100 * time.Millisecond
exponentialBackoff.MaxElapsedTime = 20 * time.Second
leaderNotSelected := errors.New("leader not selected yet")
l, err = backoff.RetryWithData(
func() (l pb.ServerAddress, err error) {
l, err = t.MaybeLeader()
if err == nil && l == "" {
err = leaderNotSelected
}
return l, err
},
exponentialBackoff)
if err == leaderNotSelected {
l = ""
}
return
return l, err
}
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
+25
View File
@@ -395,6 +395,31 @@ func (vl *VolumeLayout) setVolumeWritable(vid needle.VolumeId) bool {
return true
}
func (vl *VolumeLayout) SetVolumeReadOnly(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
if _, ok := vl.vid2location[vid]; ok {
vl.readonlyVolumes.Add(vid, dn)
return vl.removeFromWritable(vid)
}
return true
}
func (vl *VolumeLayout) SetVolumeWritable(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
if _, ok := vl.vid2location[vid]; ok {
vl.readonlyVolumes.Remove(vid, dn)
}
if vl.enoughCopies(vid) {
return vl.setVolumeWritable(vid)
}
return false
}
func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
+1 -1
View File
@@ -5,7 +5,7 @@ import (
)
var (
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.42)
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.44)
VERSION = sizeLimit + " " + VERSION_NUMBER
COMMIT = ""
)