Compare commits

..
Author SHA1 Message Date
chrislu c057ae6004 enable gocdk in the generated local image
fix https://github.com/seaweedfs/seaweedfs/issues/4256
2023-02-27 19:19:12 -08:00
chrislu 5e42b69841 add uint64 2023-02-16 21:34:23 -08:00
chrislu 59f44b70c3 tests can compile 2023-02-13 00:01:14 -08:00
98 changed files with 2330 additions and 1001 deletions

No files matched your search

+4 -4
View File
@@ -44,7 +44,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -93,7 +93,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+2 -2
View File
@@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@efb9406897e4eff1aa2c7142223b096aec90f267 # v1.22
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
+1 -1
View File
@@ -36,7 +36,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
with:
buildkitd-flags: "--debug"
-
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
with:
buildkitd-flags: "--debug"
-
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
-22
View File
@@ -1,22 +0,0 @@
name: "helm: publish charts"
on:
push:
tags:
- '*'
permissions:
contents: write
pages: write
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@master
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
-51
View File
@@ -1,51 +0,0 @@
name: "helm: lint and test charts"
on:
push:
branches: [ master ]
paths: ['k8s/**']
pull_request:
branches: [ master ]
paths: ['k8s/**']
permissions:
contents: read
jobs:
lint-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.10.0
- uses: actions/setup-python@v4
with:
python-version: '3.9'
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.1
- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --chart-dirs k8s/charts)
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
- name: Run chart-testing (lint)
run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
- name: Create kind cluster
uses: helm/kind-action@v1.5.0
- name: Run chart-testing (install)
run: ct install --all --chart-dirs k8s/charts
+2 -3
View File
@@ -8,7 +8,7 @@
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/seaweedfs)](https://artifacthub.io/packages/search?repo=seaweedfs)
![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png)
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
-->
### Gold Sponsors
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
---
@@ -66,7 +66,6 @@ Table of Contents
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
* [Compared to GlusterFS](#compared-to-glusterfs)
* [Compared to Ceph](#compared-to-ceph)
* [Compared to Minio](#compared-to-minio)
* [Dev Plan](#dev-plan)
* [Installation Guide](#installation-guide)
* [Disk Related Topics](#disk-related-topics)
+55 -56
View File
@@ -3,23 +3,22 @@ module github.com/seaweedfs/seaweedfs
go 1.20
require (
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/pubsub v1.28.0
cloud.google.com/go/storage v1.29.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.44.219
github.com/cenkalti/backoff/v4 v4.2.0
github.com/aws/aws-sdk-go v1.44.189
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.1
github.com/dustin/go-humanize v1.0.0
github.com/eapache/go-resiliency v1.3.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
github.com/eapache/queue v1.1.0 // indirect
@@ -28,11 +27,11 @@ require (
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.21.0
github.com/fclairamb/ftpserverlib v0.20.0
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.8.1
github.com/go-redsync/redsync/v4 v4.7.1
github.com/go-sql-driver/mysql v1.7.0
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
@@ -59,15 +58,15 @@ require (
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/klauspost/reedsolomon v1.11.7
github.com/klauspost/reedsolomon v1.11.6
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.7
github.com/linxGnu/grocksdb v1.7.15
github.com/linxGnu/grocksdb v1.7.14
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -79,10 +78,10 @@ require (
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.39.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.9.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.9.0 // indirect
@@ -109,75 +108,74 @@ require (
go.etcd.io/etcd/client/v3 v3.5.7
go.mongodb.org/mongo-driver v1.11.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.29.0
gocloud.dev v0.28.0
gocloud.dev/pubsub/natspubsub v0.28.0
gocloud.dev/pubsub/rabbitpubsub v0.29.0
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
golang.org/x/image v0.6.0
golang.org/x/net v0.8.0
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.6.0
golang.org/x/text v0.8.0 // indirect
golang.org/x/tools v0.6.0
gocloud.dev/pubsub/rabbitpubsub v0.27.0
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.5.0
golang.org/x/oauth2 v0.2.0 // indirect
golang.org/x/sys v0.4.0
golang.org/x/text v0.6.0 // indirect
golang.org/x/tools v0.5.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.112.0
google.golang.org/api v0.108.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
google.golang.org/grpc v1.53.0
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.52.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.3 // indirect
modernc.org/libc v1.22.2 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.21.0
modernc.org/sqlite v1.20.1
modernc.org/strutil v1.1.3
modernc.org/token v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.4.0
github.com/arangodb/go-driver v1.5.2
github.com/arangodb/go-driver v1.4.1
github.com/armon/go-metrics v0.4.1
github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
github.com/hanwen/go-fuse/v2 v2.2.0
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/rabbitmq/amqp091-go v1.7.0
github.com/rabbitmq/amqp091-go v1.5.0
github.com/schollz/progressbar/v3 v3.13.0
github.com/tikv/client-go/v2 v2.0.5
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
require (
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
github.com/aws/smithy-go v1.13.4 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.0
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
@@ -185,18 +183,18 @@ require (
github.com/fclairamb/go-log v0.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
@@ -214,6 +212,7 @@ require (
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
@@ -230,11 +229,11 @@ require (
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/term v0.4.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
+182 -319
View File
File diff suppressed because it is too large. Load diff
-16
View File
@@ -1,16 +0,0 @@
# Artifact Hub repository metadata file
#
# Some settings like the verified publisher flag or the ignored packages won't
# be applied until the next time the repository is processed. Please keep in
# mind that the repository won't be processed if it has not changed since the
# last time it was processed. Depending on the repository kind, this is checked
# in a different way. For Helm http based repositories, we consider it has
# changed if the `index.yaml` file changes. For git based repositories, it does
# when the hash of the last commit in the branch you set up changes. This does
# NOT apply to ownership claim operations, which are processed immediately.
#
repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
# owners: # (optional, used to claim repository ownership)
# - name: username
# email: email
-81
View File
@@ -1,81 +0,0 @@
# SEAWEEDFS - helm chart (2.x+)
## Getting Started
### Add the helm repo
`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
### Install the helm chart
`helm install seaweedfs seaweedfs/seaweedfs`
### (Recommended) Provide `values.yaml`
`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
## Info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
## Prerequisites
### Database
leveldb is the default database this only supports one filer replica.
To have multiple filers a external datastore is recommened.
Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
```
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
### Node Labels
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
Here is an example:
* s3/filer/master needs the label **sw-backend=true**
* volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s:
```
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
```
on production k8s deployment you will want each pod to have a different host,
especially the volume server and the masters, all pods (master/volume/filer)
should have anti-affinity rules to disallow running multiple component pods on the same host.
If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
```affinity: ""```
## PVC - storage class ###
On the volume stateful set added support for k8s PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class
for your deployment.
## current instances config (AIO):
1 instance for each type (master/filer+s3/volume)
You can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding labels if applicable.
Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
File renamed without changes.
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.44"
version: "3.44"
appVersion: "3.42"
version: "3.42"
+62
View File
@@ -0,0 +1,62 @@
# SEAWEEDFS - helm chart (2.x)
## Info:
* master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
and backup/HA memsql can provide.
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
with ENV.
* cert config exists and can be enabled, but not been tested.
## Prerequisites
### Database
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS `filemeta` (
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
```
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
### Node Labels
Kubernetes node have labels which help to define which node(Host) will run which pod:
* s3/filer/master needs the label **sw-backend=true**
* volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s:
```
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
```
on production k8s deployment you will want each pod to have a different host,
especially the volume server & the masters, currently all pods (master/volume/filer)
have anti-affinity rule to disallow running multiple pod type on the same host.
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
please set/update the corresponding affinity rule in values.yaml to an empty one:
```affinity: ""```
## PVC - storage class ###
on the volume stateful set added support for K8S PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class
for your deployment.
## current instances config (AIO):
1 instance for each type (master/filer+s3/volume)
you can update the replicas count for each node type in values.yaml,
need to add more nodes with the corresponding labels.
most of the configuration are available through values.yaml
@@ -97,12 +97,6 @@ spec:
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
valueFrom: {{ toYaml $value | nindent 16 }}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -168,7 +162,7 @@ spec:
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
{{- end }}
{{- end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name: seaweedfs-filer-log-volume
mountPath: "/logs/"
@@ -133,8 +133,8 @@ spec:
{{- if .Values.master.garbageThreshold }}
-garbageThreshold={{ .Values.master.garbageThreshold }} \
{{- end }}
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name : data-{{ .Release.Namespace }}
mountPath: /data
@@ -105,7 +105,7 @@ spec:
{{- if .Values.s3.auditLogConfig }}
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
{{- end }}
-filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
volumeMounts:
- name: logs
mountPath: "/logs/"
@@ -138,9 +138,9 @@ spec:
-fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
{{- end }}
-minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
-compactionMBps={{ .Values.volume.compactionMBps }} \
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
volumeMounts:
- name: data
mountPath: "{{ .Values.volume.dir }}/"
@@ -187,7 +187,7 @@ spec:
port: {{ .Values.volume.port }}
scheme: HTTP
initialDelaySeconds: 15
periodSeconds: 15
periodSeconds: 90
successThreshold: 1
failureThreshold: 100
timeoutSeconds: 30
@@ -5,7 +5,7 @@ global:
repository: ""
imageName: chrislusf/seaweedfs
imagePullPolicy: IfNotPresent
imagePullSecrets: ""
imagePullSecrets: imagepullsecret
restartPolicy: Always
loggingLevel: 1
enableSecurity: false
@@ -22,8 +22,8 @@ global:
replicationPlacment: "001"
extraEnvironmentVars:
WEED_CLUSTER_DEFAULT: "sw"
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
image:
registry: ""
@@ -44,11 +44,11 @@ master:
volumePreallocate: false
volumeSizeLimitMB: 1000
loggingOverrideLevel: null
# number of seconds between heartbeats, default 5
#number of seconds between heartbeats, default 5
pulseSeconds: null
# threshold to vacuum and reclaim spaces, default 0.3 (30%)
#threshold to vacuum and reclaim spaces, default 0.3 (30%)
garbageThreshold: null
# Prometheus push interval in seconds, default 15
#Prometheus push interval in seconds, default 15
metricsIntervalSec: 15
# replication type is XYZ:
# X number of replica in other data centers
@@ -112,10 +112,10 @@ master:
# nodeSelector labels for master pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
nodeSelector: |
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
# used to assign priority to master pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -241,7 +241,7 @@ volume:
# should map directly to the value of the resources field for a PodSpec,
# formatted as a multi-line string. By default no direct resource request
# is made.
resources: {}
resources: null
# Toleration Settings for server pods
# This should be a multi-line string matching the Toleration array
@@ -251,10 +251,10 @@ volume:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
nodeSelector: |
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-volume: "true"
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-volume: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -358,10 +358,10 @@ filer:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
nodeSelector: |
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -389,7 +389,7 @@ filer:
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
extraEnvironmentVars:
WEED_MYSQL_ENABLED: "false"
WEED_MYSQL_ENABLED: "true"
WEED_MYSQL_HOSTNAME: "mysql-db-host"
WEED_MYSQL_PORT: "3306"
WEED_MYSQL_DATABASE: "sw_database"
@@ -400,29 +400,17 @@ filer:
# enable usage of memsql as filer backend
WEED_MYSQL_INTERPOLATEPARAMS: "true"
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
WEED_LEVELDB2_ENABLED: "true"
WEED_LEVELDB2_ENABLED: "false"
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
# directories under this folder will be automatically creating a separate bucket
WEED_FILER_BUCKETS_FOLDER: "/buckets"
# secret env variables
secretExtraEnvironmentVars: []
# WEED_POSTGRES_USERNAME:
# secretKeyRef:
# name: postgres-credentials
# key: username
# WEED_POSTGRES_PASSWORD:
# secretKeyRef:
# name: postgres-credentials
# key: password
s3:
enabled: true
port: 8333
# allow empty folders
#allow empty folders
allowEmptyFolder: false
# Suffix of the host name, {bucket}.{domainName}
domainName: ""
@@ -442,7 +430,7 @@ s3:
port: 8333
metricsPort: 9327
loggingOverrideLevel: null
# allow empty folders
#allow empty folders
allowEmptyFolder: true
# enable user & permission to s3 (need to inject to all services)
enableAuth: false
@@ -461,7 +449,7 @@ s3:
# should map directly to the value of the resources field for a PodSpec,
# formatted as a multi-line string. By default no direct resource request
# is made.
resources: {}
resources: null
# Toleration Settings for server pods
# This should be a multi-line string matching the Toleration array
@@ -471,10 +459,10 @@ s3:
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
# Example:
nodeSelector: |
beta.kubernetes.io/arch: amd64
# nodeSelector: |
# sw-backend: "true"
# beta.kubernetes.io/arch: amd64
nodeSelector: |
sw-backend: "true"
# used to assign priority to server pods
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
@@ -491,5 +479,5 @@ certificates:
ipAddresses: []
keyAlgorithm: rsa
keySize: 2048
duration: 2160h # 90d
renewBefore: 360h # 15d
duration: 2160h # 90d
renewBefore: 360h # 15d
-4
View File
@@ -5,10 +5,6 @@
<artifactId>copier</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>18</maven.compiler.source>
<maven.compiler.target>18</maven.compiler.target>
</properties>
<name>copier</name>
<url>http://maven.apache.org</url>
@@ -10,8 +10,6 @@ import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
@@ -44,19 +42,6 @@ public class HighLevelMultipartUpload {
.withS3Client(s3Client)
.build();
if (!s3Client.doesBucketExistV2(bucketName)) {
// Because the CreateBucketRequest object doesn't specify a region, the
// bucket is created in the region specified in the client.
s3Client.createBucket(new CreateBucketRequest(bucketName));
// Verify that the bucket was created by retrieving it and checking its location.
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
System.out.println("Bucket location: " + bucketLocation);
} else {
System.out.println("Bucket already exists");
}
// TransferManager processes all transfers asynchronously,
// so this call returns immediately.
Upload upload = tm.upload(bucketName, keyName, file);
-8
View File
@@ -60,7 +60,6 @@ type FilerOptions struct {
localSocket *string
showUIDirectoryDelete *bool
downloadMaxMBps *int
diskType *string
}
func init() {
@@ -90,7 +89,6 @@ func init() {
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
// start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -185,11 +183,6 @@ func runFiler(cmd *Command, args []string) bool {
if *filerStartWebDav {
filerWebDavOptions.filer = &filerAddress
if *filerWebDavOptions.disk == "" {
filerWebDavOptions.disk = f.diskType
}
go func(delay time.Duration) {
time.Sleep(delay * time.Second)
filerWebDavOptions.startWebDav()
@@ -250,7 +243,6 @@ func (fo *FilerOptions) startFiler() {
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
DiskType: *fo.diskType,
})
if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err)
+6 -7
View File
@@ -2,14 +2,13 @@ package command
import (
"fmt"
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
"strings"
"time"
hashicorpRaft "github.com/hashicorp/raft"
"golang.org/x/exp/slices"
"github.com/gorilla/mux"
@@ -164,11 +163,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
RaftResumeState: *masterOption.raftResumeState,
HeartbeatInterval: *masterOption.heartbeatInterval,
ElectionTimeout: *masterOption.electionTimeout,
RaftBootstrap: *masterOption.raftBootstrap,
RaftBootstrap: *m.raftBootstrap,
}
var raftServer *weed_server.RaftServer
var err error
if *masterOption.raftHashicorp {
if *m.raftHashicorp {
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
glog.Fatalf("NewHashicorpRaftServer: %s", err)
}
@@ -181,7 +180,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
if *masterOption.raftHashicorp {
if *m.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
}
// starting grpc server
@@ -192,7 +191,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
if *masterOption.raftHashicorp {
if *m.raftHashicorp {
raftServer.TransportManager.Register(grpcS)
} else {
protobuf.RegisterRaftServer(grpcS, raftServer)
@@ -205,7 +204,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go grpcS.Serve(grpcL)
timeSleep := 1500 * time.Millisecond
if !*masterOption.raftHashicorp {
if !*m.raftHashicorp {
go func() {
time.Sleep(timeSleep)
-5
View File
@@ -97,7 +97,6 @@ func init() {
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
@@ -116,7 +115,6 @@ func init() {
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
@@ -255,9 +253,6 @@ func runServer(cmd *Command, args []string) bool {
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
if *isStartingFiler {
if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
filerOptions.diskType = serverOptions.v.diskType
}
go func() {
time.Sleep(1 * time.Second)
filerOptions.startFiler()
+4 -3
View File
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -214,7 +215,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
buf, err := io.ReadAll(res.Body)
buf, err := ioutil.ReadAll(res.Body)
if err != nil {
return Release{}, err
}
@@ -264,7 +265,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
buf, err := io.ReadAll(res.Body)
buf, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
@@ -336,7 +337,7 @@ func extractToFile(buf []byte, filename, target string) error {
// Write everything to a temp file
dir := filepath.Dir(target)
new, err := os.CreateTemp(dir, "weed")
new, err := ioutil.TempFile(dir, "weed")
if err != nil {
return err
}
+6
View File
@@ -0,0 +1,6 @@
all: gen
.PHONY : gen
gen:
protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint16 struct {
}
const SIZE_Uint16 = 2
func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
return binary.BigEndian.Uint16(buf)
}
return 0
}
func WriteUint16s(buf []byte, data []uint16) (err error) {
off := 0
size := len(data)
if len(buf) < size<<1 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint16(buf[off:], dat)
off += SIZE_Uint16
}
return nil
}
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint32 struct {
}
const SIZE_Uint32 = 4
func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
return binary.BigEndian.Uint32(buf)
}
return 0
}
func WriteUint32s(buf []byte, data []uint32) (err error) {
off := 0
size := len(data)
if len(buf) < size<<2 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint32(buf[off:], dat)
off += SIZE_Uint32
}
return nil
}
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint64 struct {
}
const SIZE_Uint64 = 8
func (c *ColumnUint64) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint64 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
return binary.BigEndian.Uint64(buf)
}
return 0
}
func WriteUint64s(buf []byte, data []uint64) (err error) {
off := 0
size := len(data)
if len(buf) < size<<3 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint64(buf[off:], dat)
off += SIZE_Uint64
}
return nil
}
+103
View File
@@ -0,0 +1,103 @@
syntax = "proto3";
package columnar_pb;
option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb";
message FileId {
uint32 volume_id = 1;
uint64 file_key = 2;
fixed32 cookie = 3;
}
enum LogicalType {
Uint8 = 0;
Uint16 = 1;
Float32 = 4;
}
message ColumnUint16 {
uint32 base = 1;
uint32 min = 3;
uint32 max = 4;
}
message ColumnUint32 {
uint32 base = 1;
uint32 min = 3;
uint32 max = 4;
}
message ColumnFloat32 {
uint32 min = 3;
uint32 max = 4;
}
message ColumnSplit {
// The ids of the fields/columns in this file
int32 field_id = 1;
FileId file_id = 2;
int64 row_offset = 3;
int32 row_count = 4;
oneof storage_type {
ColumnUint16 meta_uint16 = 8;
ColumnUint32 meta_uint32 = 9;
ColumnFloat32 meta_float32 = 10;
}
}
message Snapshot {
// All fields of the dataset, including the nested fields.
repeated Field fields = 1;
repeated string data_files = 2;
// Snapshot version number.
uint64 version = 3;
}
message DataFile {
repeated int32 field_ids = 1;
repeated RowGroup row_groups = 2;
}
message RowGroup {
int64 row_offset = 1;
int32 row_count = 2;
repeated ColumnSplit column_splits = 3;
}
// Field metadata for a column.
message Field {
enum Type {
PARENT = 0;
REPEATED = 1;
LEAF = 2;
}
Type type = 1;
// Fully qualified name.
string name = 2;
/// Field Id.
int32 id = 3;
/// Parent Field ID. If not set, this is a top-level column.
int32 parent_id = 4;
// Logical types, support parameterized Arrow Type.
LogicalType logical_type = 5;
// If this field is nullable.
bool nullable = 6;
}
message AnyValue {
oneof value {
bytes bytes_value = 1;
bool bool_value = 2;
uint64 int64_value = 3;
uint32 int32_value = 4;
double double_value = 5;
}
}
File diff suppressed because it is too large. Load diff
+91
View File
@@ -0,0 +1,91 @@
package data
import "fmt"
type Datum interface {
Compare(other Datum) (int, error)
}
type Datums []Datum
type DUint16 uint16
type DUint32 uint32
type DUint64 uint64
type dNull struct{}
var (
DNull Datum = dNull{}
)
func (d dNull) Compare(other Datum) (int, error) {
if other == DNull {
return 0, nil
}
return -1, nil
}
func NewDUint16(d DUint16) *DUint16 {
return &d
}
func NewDUint32(d DUint32) *DUint32 {
return &d
}
func NewDUint64(d DUint64) *DUint64 {
return &d
}
func (d *DUint16) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint16
switch t := other.(type) {
case *DUint16:
otherV = *t
default:
return 0, fmt.Errorf("unsupported")
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
func (d *DUint32) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint32
switch t := other.(type) {
case *DUint32:
otherV = *t
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
func (d *DUint64) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint64
switch t := other.(type) {
case *DUint64:
otherV = *t
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
+259
View File
@@ -0,0 +1,259 @@
package data
import (
"encoding/binary"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"testing"
)
func TestRead(t *testing.T) {
x := make([]uint16, 128)
y := make([]uint32, 128)
z := make([]uint64, 128)
for i := range x {
x[i] = uint16(i)
}
for i := range y {
y[i] = uint32(i * 2)
}
for i := range z {
z[i] = uint64(i * 3)
}
xbuf := make([]byte, len(x)*SIZE_Uint16)
ybuf := make([]byte, len(x)*SIZE_Uint32)
zbuf := make([]byte, len(x)*SIZE_Uint64)
WriteUint16s(xbuf, x)
WriteUint32s(ybuf, y)
WriteUint64s(zbuf, z)
df := &DataFile{
xbuf: xbuf,
ybuf: ybuf,
zbuf: zbuf,
xLen: len(xbuf),
yLen: len(ybuf),
zLen: len(zbuf),
xReaderAt: util.NewBytesReader(xbuf),
yReaderAt: util.NewBytesReader(ybuf),
zReaderAt: util.NewBytesReader(zbuf),
}
dataLayout := make(map[FieldName]DataLayout)
dataLayout["x"] = DataLayout{
LayoutType: Uint16,
SortType: Unsorted,
}
dataLayout["y"] = DataLayout{
LayoutType: Uint32,
SortType: Unsorted,
}
dataLayout["z"] = DataLayout{
LayoutType: Uint64,
SortType: Unsorted,
}
rows, err := df.ReadRows("x", dataLayout, Equal, NewDUint16(65))
if err != nil {
fmt.Printf("err: %v", err)
return
}
for _, row := range rows {
fmt.Printf("row %d width %d ", row.index, len(row.Datums))
for i, d := range row.Datums {
fmt.Printf("%d: %v ", i, d)
}
fmt.Println()
}
}
type Operator int32
type LayoutType int32
type SortType int32
const (
Equal Operator = 0
GreaterThan
GreaterOrEqual
LessThan
LessOrEqual
Uint16 LayoutType = 0
Uint32 = 1
Uint64 = 2
Unsorted SortType = 0
Ascending
Descending
)
type DataFile struct {
xbuf []byte
ybuf []byte
zbuf []byte
xReaderAt io.ReaderAt
yReaderAt io.ReaderAt
zReaderAt io.ReaderAt
xLen int
yLen int
zLen int
}
type DataLayout struct {
LayoutType
SortType
}
type FieldName string
func (d *DataFile) ReadRows(field FieldName, layout map[FieldName]DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
if field == "x" {
rows, err = pushDownReadRows(d.xReaderAt, d.xLen, layout[field], op, operand)
if err != nil {
return
}
err = hydrateRows(d.yReaderAt, d.yLen, layout["y"], rows)
if err != nil {
return
}
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
if err != nil {
return
}
}
if field == "y" {
rows, err = pushDownReadRows(d.yReaderAt, d.yLen, layout[field], op, operand)
if err != nil {
return
}
err = hydrateRows(d.xReaderAt, d.xLen, layout["x"], rows)
if err != nil {
return
}
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
if err != nil {
return
}
}
return
}
type Row struct {
index int
Datums
}
func pushDownReadRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
if layout.LayoutType == Uint16 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint16)
for i := 0; i < dataLen; i += SIZE_Uint16 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint16 && err == nil {
d := NewDUint16(DUint16(binary.BigEndian.Uint16(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(1)
rows = append(rows, &Row{
index: i / SIZE_Uint16,
Datums: []Datum{d},
})
}
}
}
}
}
if layout.LayoutType == Uint32 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint32)
for i := 0; i < dataLen; i += SIZE_Uint32 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint32 && err == nil {
d := NewDUint32(DUint32(binary.BigEndian.Uint32(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(2)
rows = append(rows, &Row{
index: i / SIZE_Uint32,
Datums: []Datum{d},
})
}
}
}
}
}
if layout.LayoutType == Uint64 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint64)
for i := 0; i < dataLen; i += SIZE_Uint64 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint64 && err == nil {
d := NewDUint64(DUint64(binary.BigEndian.Uint64(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(2)
rows = append(rows, &Row{
index: i / SIZE_Uint64,
Datums: []Datum{d},
})
}
}
}
}
}
return
}
func hydrateRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, rows []*Row) (err error) {
if layout.LayoutType == Uint16 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint16)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
t := binary.BigEndian.Uint16(buf)
d := NewDUint16(DUint16(t))
println(3, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
if layout.LayoutType == Uint32 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint32)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
t := binary.BigEndian.Uint32(buf)
d := NewDUint32(DUint32(t))
println(4, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
if layout.LayoutType == Uint64 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint64)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
t := binary.BigEndian.Uint64(buf)
d := NewDUint64(DUint64(t))
println(5, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
return
}
+22 -15
View File
@@ -300,23 +300,30 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
return nil
}
// glog.V(2).Infof("c.chunkView: %v buffer:[%d,%d) offset:%d totalSize:%d", c.chunkView, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)), offset, c.totalSize)
if c.chunkView == nil {
return io.EOF
}
// find a possible chunk view
p := c.chunkView
for p != nil {
chunk := p.Value
// glog.V(2).Infof("prepareBufferFor check chunk:[%d,%d)", chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
c.chunkView = p
return c.fetchChunkToBuffer(chunk)
}
// fmt.Printf("fetch for offset %d\n", offset)
c.chunkView = c.chunkView.Next
if c.chunkView == nil {
return io.EOF
}
// positioning within the new chunk
chunk := c.chunkView.Value
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
return c.fetchChunkToBuffer(chunk)
}
if offset < c.bufferOffset {
p = p.Prev
} else {
p = p.Next
} else {
for p := c.head; p != nil; p = p.Next {
chunk = p.Value
if insideChunk(offset, chunk) {
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
return c.fetchChunkToBuffer(chunk)
}
}
}
}
+47 -47
View File
@@ -1,47 +1,47 @@
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}
+1 -1
View File
@@ -53,7 +53,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
return 0, 0, io.EOF
}
if offset < int64(len(entry.Content)) {
if offset+int64(len(buff)) <= int64(len(entry.Content)) {
totalRead := copy(buff, entry.Content[offset:])
glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
return int64(totalRead), 0, nil
+7 -10
View File
@@ -3,16 +3,14 @@ package mount
import (
"context"
"fmt"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"strings"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
/** Create a directory
@@ -56,10 +54,9 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
defer wfs.mapPbIdFromFilerToLocal(newEntry)
request := &filer_pb.CreateEntryRequest{
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
}
glog.V(1).Infof("mkdir: %v", request)
+3 -6
View File
@@ -2,14 +2,12 @@ package mount
import (
"context"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"syscall"
"time"
)
/*
@@ -74,8 +72,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
HardLinkId: oldEntry.HardLinkId,
HardLinkCounter: oldEntry.HardLinkCounter,
},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
Signatures: []int32{wfs.signature},
}
// apply changes to the filer, and also apply to local metaCache
+4 -7
View File
@@ -3,15 +3,13 @@ package mount
import (
"context"
"fmt"
"os"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"syscall"
"time"
)
/** Create a symbolic link */
@@ -44,8 +42,7 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
SymlinkTarget: target,
},
},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
Signatures: []int32{wfs.signature},
}
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+1 -1
View File
@@ -1,6 +1,6 @@
package s3_constants
// Amazon S3 predefined groups
//Amazon S3 predefined groups
var (
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+40 -48
View File
@@ -146,65 +146,57 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
// check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
for {
empty := true
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
empty = false
if entry.IsDirectory {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: "STANDARD",
})
cursor.maxKeys--
}
} else {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: StorageClass(storageClass),
StorageClass: "STANDARD",
})
cursor.maxKeys--
}
})
if doErr != nil {
return doErr
}
if cursor.isTruncated {
if requestDir != "" {
nextMarker = requestDir + "/" + nextMarker
}
break
} else if empty {
nextMarker = ""
break
} else {
// start next loop
marker = nextMarker
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: StorageClass(storageClass),
})
cursor.maxKeys--
}
})
if doErr != nil {
return doErr
}
if !cursor.isTruncated {
nextMarker = ""
} else {
if requestDir != "" {
nextMarker = requestDir + "/" + nextMarker
}
}
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
"google.golang.org/grpc/security/advancedtls"
"os"
"io/ioutil"
"strings"
"time"
@@ -136,7 +136,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
}
func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
clientCerts, err := os.ReadFile(clientCertFile)
clientCerts, err := ioutil.ReadFile(clientCertFile)
if err != nil {
glog.Fatal(err)
}
-4
View File
@@ -295,10 +295,6 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
if req.DiskType == "" {
req.DiskType = fs.option.DiskType
}
so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack, req.DataNode)
if err != nil {
glog.V(3).Infof("AssignVolume: %v", err)
+1 -2
View File
@@ -154,11 +154,10 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
Offset: localOffset,
Size: uint64(size),
ModifiedTsNs: time.Now().UnixNano(),
ModifiedTsNs: time.Now().Unix(),
ETag: etag,
Fid: &filer_pb.FileId{
VolumeId: uint32(fileId.VolumeId),
+3 -1
View File
@@ -69,7 +69,6 @@ type FilerOption struct {
ConcurrentUploadLimit int64
ShowUIDirectoryDelete bool
DownloadMaxBytesPs int64
DiskType string
}
type FilerServer struct {
@@ -195,6 +194,9 @@ func (fs *FilerServer) checkWithMaster() {
return fmt.Errorf("get master %s configuration: %v", master, err)
}
fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
if fs.option.DefaultReplication == "" {
fs.option.DefaultReplication = resp.DefaultReplication
}
return nil
})
if readErr == nil {
@@ -27,7 +27,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
limit, limit_err := strconv.Atoi(r.FormValue("limit"))
if limit_err != nil {
limit = fs.option.DirListingLimit
limit = 100
}
lastFileName := r.FormValue("lastFileName")
+1 -7
View File
@@ -4,13 +4,12 @@ import (
"context"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"net/http"
"os"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -98,11 +97,6 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
return
}
// When DiskType is empty,use filer's -disk
if so.DiskType == "" {
so.DiskType = fs.option.DiskType
}
if query.Has("mv.from") {
fs.move(ctx, w, r, so)
} else {
@@ -49,7 +49,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
writeJsonError(w, r, 499, err)
} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
} else if strings.HasSuffix(err.Error(), "is a file") {
writeJsonError(w, r, http.StatusConflict, err)
} else {
writeJsonError(w, r, http.StatusInternalServerError, err)
+2 -9
View File
@@ -309,19 +309,12 @@ func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.V
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(req.ReplicaPlacement))
vl := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, needle.LoadTTLFromUint32(req.Ttl), types.ToDiskType(req.DiskType))
dataNodes := ms.Topo.Lookup(req.Collection, needle.VolumeId(req.VolumeId))
if req.IsReadonly {
for _, dn := range dataNodes {
if dn.Ip == req.Ip && dn.Port == int(req.Port) {
}
}
}
for _, dn := range dataNodes {
if dn.Ip == req.Ip && dn.Port == int(req.Port) {
if req.IsReadonly {
vl.SetVolumeReadOnly(dn, needle.VolumeId(req.VolumeId))
vl.SetVolumeUnavailable(dn, needle.VolumeId(req.VolumeId))
} else {
vl.SetVolumeWritable(dn, needle.VolumeId(req.VolumeId))
vl.SetVolumeAvailable(dn, needle.VolumeId(req.VolumeId), false)
}
}
}
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
transport "github.com/Jille/raft-grpc-transport"
"io"
"io/ioutil"
"math/rand"
"os"
"path"
@@ -89,7 +90,7 @@ func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
}
func (s *StateMachine) Restore(r io.ReadCloser) error {
b, err := io.ReadAll(r)
b, err := ioutil.ReadAll(r)
if err != nil {
return err
}
+20 -65
View File
@@ -12,11 +12,9 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/util"
"golang.org/x/exp/slices"
"io"
"math"
"strings"
"sync"
"time"
)
@@ -26,14 +24,10 @@ func init() {
type commandFsVerify struct {
env *CommandEnv
volumeServers []pb.ServerAddress
volumeIds map[uint32][]pb.ServerAddress
verbose *bool
concurrency *int
modifyTimeAgoAtSec int64
writer io.Writer
waitChan map[string]chan struct{}
waitChanLock sync.RWMutex
}
func (c *commandFsVerify) Name() string {
@@ -51,10 +45,10 @@ func (c *commandFsVerify) Help() string {
func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
c.env = commandEnv
c.writer = writer
fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server")
if err = fsVerifyCommand.Parse(args); err != nil {
return err
@@ -66,29 +60,13 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
}
c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
c.volumeIds = make(map[uint32][]pb.ServerAddress)
c.waitChan = make(map[string]chan struct{})
c.volumeServers = []pb.ServerAddress{}
defer func() {
c.modifyTimeAgoAtSec = 0
c.volumeIds = nil
c.waitChan = nil
c.volumeServers = nil
}()
if err := c.collectVolumeIds(); err != nil {
return parseErr
}
if *c.concurrency > 0 {
for _, volumeServer := range c.volumeServers {
volumeServerStr := string(volumeServer)
c.waitChan[volumeServerStr] = make(chan struct{}, *c.concurrency)
defer close(c.waitChan[volumeServerStr])
}
}
fCount, eConut, terr := c.verifyTraverseBfs(path)
if terr == nil {
fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut)
}
@@ -98,6 +76,7 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
}
func (c *commandFsVerify) collectVolumeIds() error {
c.volumeIds = make(map[uint32][]pb.ServerAddress)
topologyInfo, _, err := collectTopologyInfo(c.env, 0)
if err != nil {
return err
@@ -105,19 +84,15 @@ func (c *commandFsVerify) collectVolumeIds() error {
eachDataNode(topologyInfo, func(dc string, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
for _, diskInfo := range nodeInfo.DiskInfos {
for _, vi := range diskInfo.VolumeInfos {
volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
if !slices.Contains(c.volumeServers, volumeServer) {
c.volumeServers = append(c.volumeServers, volumeServer)
}
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], pb.NewServerAddressFromDataNode(nodeInfo))
}
}
})
return nil
}
func (c *commandFsVerify) verifyEntry(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error {
err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption,
func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.ServerAddress) error {
err := operation.WithVolumeServerClient(false, *volumeServer, c.env.option.GrpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeNeedleStatus(context.Background(),
&volume_server_pb.VolumeNeedleStatusRequest{
@@ -129,6 +104,9 @@ func (c *commandFsVerify) verifyEntry(volumeServer pb.ServerAddress, fileId *fil
if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
return err
}
if *c.verbose {
fmt.Fprintf(c.writer, ".")
}
return nil
}
@@ -162,50 +140,27 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
func(outputChan chan interface{}) {
for itemEntry := range outputChan {
i := itemEntry.(*ItemEntry)
itemPath := string(i.path)
fileMsg := fmt.Sprintf("file:%s", itemPath)
errItem := make(map[string]error)
errItemLock := sync.RWMutex{}
fileMsg := fmt.Sprintf("file:%s needle status ", i.path)
if *c.verbose {
fmt.Fprintf(c.writer, fileMsg)
fileMsg = ""
}
for _, chunk := range i.chunks {
if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
for _, volumeServer := range volumeIds {
if *c.concurrency == 0 {
if err = c.verifyEntry(volumeServer, chunk.Fid); err != nil {
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
}
continue
if err = c.verifyEntry(chunk.Fid, &volumeServer); err != nil {
fmt.Fprintf(c.writer, "%sfailed verify %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
break
}
c.waitChanLock.RLock()
waitChan, ok := c.waitChan[string(volumeServer)]
c.waitChanLock.RUnlock()
if !ok {
fmt.Fprintf(c.writer, "%s failed to get channel for %s chunk: %d:%d: %+v\n",
string(volumeServer), fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
continue
}
waitChan <- struct{}{}
go func(fId *filer_pb.FileId, path string, volumeServer pb.ServerAddress, msg string) {
if err = c.verifyEntry(volumeServer, fId); err != nil {
errItemLock.Lock()
errItem[path] = err
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
msg, fId.VolumeId, fId.FileKey, err)
errItemLock.Unlock()
}
<-waitChan
}(chunk.Fid, itemPath, volumeServer, fileMsg)
}
} else {
err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
fmt.Fprintf(c.writer, "%s %d:%d: %+v\n",
fmt.Fprintf(c.writer, "%sfailed verify chunk %d:%d: %+v\n",
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
break
}
}
errItemLock.RLock()
err, _ = errItem[itemPath]
errItemLock.RUnlock()
if err != nil {
errCount++
@@ -213,7 +168,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
}
if *c.verbose {
fmt.Fprintf(c.writer, "%s needles:%d verifed\n", fileMsg, len(i.chunks))
fmt.Fprintf(c.writer, " verifed\n")
}
fileCount++
}
+1 -1
View File
@@ -63,7 +63,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
collectionSize = collectionInfo.Size
fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
}
fmt.Fprintf(writer, " %s\tsize:%.0f\tchunk:%.0f", entry.Name, collectionSize, fileCount)
fmt.Fprintf(writer, " %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
if entry.Quota > 0 {
fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
}
+1 -1
View File
@@ -330,7 +330,7 @@ func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*Vol
}
if candidateVolume.RemoteStorageName != "" {
return false, fmt.Errorf("does not move volume in remove storage")
return false, nil
}
if candidateVolume.ReplicaPlacement > 0 {
+1 -1
View File
@@ -58,7 +58,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'")
skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes")
noDelete := volFixReplicationCommand.Bool("noDelete", false, "Do not delete over-replicated volumes, only fix under-replication")
retryCount := volFixReplicationCommand.Int("retry", 5, "how many times to retry")
retryCount := volFixReplicationCommand.Int("retry", 0, "how many times to retry")
volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle")
if err = volFixReplicationCommand.Parse(args); err != nil {
+5 -5
View File
@@ -20,6 +20,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
@@ -163,7 +164,6 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
if *c.findMissingChunksInFiler {
// collect all filer file ids and paths
if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectCutoffFromAtNs); err != nil {
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
}
@@ -409,7 +409,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
}
buf.Write(resp.FileContent)
}
if !vinfo.isReadOnly {
if vinfo.isReadOnly == false {
index, err := idx.FirstInvalidIndex(buf.Bytes(),
func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
@@ -419,12 +419,12 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
Size: int32(size),
})
if err != nil {
return false, fmt.Errorf("read needle meta with id %d from volume %d: %v", key, volumeId, err)
return false, fmt.Errorf("to read needle meta with id %d from volume %d with error %v", key, volumeId, err)
}
return resp.AppendAtNs <= cutoffFrom, nil
})
if err != nil {
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v\n", volumeId, err)
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
} else {
buf.Truncate(index * types.NeedleMapEntrySize)
}
@@ -535,7 +535,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
}
defer resp.Body.Close()
_, err = io.ReadAll(resp.Body)
_, err = ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Fprintf(c.writer, "DELETE response error: %v\n", err)
}
+17 -15
View File
@@ -19,8 +19,8 @@ func init() {
type commandVolumeServerEvacuate struct {
topologyInfo *master_pb.TopologyInfo
targetServer *string
volumeRack *string
targetServer string
volumeRack string
}
func (c *commandVolumeServerEvacuate) Name() string {
@@ -49,8 +49,8 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
c.volumeRack = vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
c.targetServer = vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
volumeRack := vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
targetServer := vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
retryCount := vsEvacuateCommand.Int("retry", 0, "how many times to retry")
@@ -63,10 +63,15 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
return
}
if *volumeServer == "" && *c.volumeRack == "" {
if *volumeServer == "" && *volumeRack == "" {
return fmt.Errorf("need to specify volume server by -node=<host>:<port> or source rack")
}
if *targetServer != "" {
c.targetServer = *targetServer
}
if *volumeRack != "" {
c.volumeRack = *volumeRack
}
for i := 0; i < *retryCount+1; i++ {
if err = c.volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer); err == nil {
return nil
@@ -88,9 +93,6 @@ func (c *commandVolumeServerEvacuate) volumeServerEvacuate(commandEnv *CommandEn
if err != nil {
return err
}
defer func() {
c.topologyInfo = nil
}()
if err := c.evacuateNormalVolumes(commandEnv, volumeServer, skipNonMoveable, applyChange, writer); err != nil {
return err
@@ -235,14 +237,14 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][
func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, thisServer string) (thisNodes []*Node, otherNodes []*Node) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (*c.volumeRack != "" && node.rack == *c.volumeRack) {
if node.info.Id == thisServer || (c.volumeRack != "" && node.rack == c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if *c.volumeRack != "" && *c.volumeRack == node.rack {
if c.volumeRack != "" && c.volumeRack == node.rack {
continue
}
if *c.targetServer != "" && *c.targetServer != node.info.Id {
if c.targetServer != "" && c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)
@@ -252,14 +254,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this
func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) {
if node.info.Id == thisServer || (c.volumeRack != "" && string(node.rack) == c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if *c.volumeRack != "" && *c.volumeRack == string(node.rack) {
if c.volumeRack != "" && c.volumeRack == string(node.rack) {
continue
}
if *c.targetServer != "" && *c.targetServer != node.info.Id {
if c.targetServer != "" && c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)
@@ -6,12 +6,7 @@ import (
)
func TestVolumeServerEvacuate(t *testing.T) {
targetServer := ""
volumeRack := "rack1"
c := commandVolumeServerEvacuate{
targetServer: &targetServer,
volumeRack: &volumeRack,
}
c := commandVolumeServerEvacuate{}
c.topologyInfo = parseOutput(topoData)
volumeServer := "192.168.1.4:8080"
-7
View File
@@ -3,7 +3,6 @@ package shell
import (
"context"
"flag"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb"
"io"
@@ -42,12 +41,6 @@ func (c *commandVolumeUnmount) Do(args []string, commandEnv *CommandEnv, writer
if err = volUnmountCommand.Parse(args); err != nil {
return nil
}
if *nodeStr == "" {
return fmt.Errorf("-node option is required")
}
if *volumeIdInt == 0 {
return fmt.Errorf("-volumeId option is required")
}
if err = commandEnv.confirmIsLocked(args); err != nil {
return
+2 -10
View File
@@ -1,6 +1,7 @@
package stats
import (
"fmt"
"log"
"net"
"net/http"
@@ -281,21 +282,12 @@ func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
}
}
func JoinHostPort(host string, port int) string {
portStr := strconv.Itoa(port)
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
return host + ":" + portStr
}
return net.JoinHostPort(host, portStr)
}
func StartMetricsServer(ip string, port int) {
if port == 0 {
return
}
http.Handle("/metrics", promhttp.HandlerFor(Gather, promhttp.HandlerOpts{}))
log.Fatal(http.ListenAndServe(JoinHostPort(ip, port), nil))
log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil))
}
func SourceName(port uint32) string {
+1 -5
View File
@@ -66,12 +66,8 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
if err != nil {
return err
}
var index int
if size.IsValid() {
index, err = n.readNeedleDataVersion2NonData(metaSlice)
}
index, err = n.readNeedleDataVersion2NonData(metaSlice)
n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize]))
if version == Version3 {
n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize])
+1 -1
View File
@@ -16,7 +16,7 @@ func TestFastLoadingNeedleMapMetrics(t *testing.T) {
for i := 0; i < 10000; i++ {
nm.Put(Uint64ToNeedleId(uint64(i+1)), Uint32ToOffset(uint32(0)), Size(1))
if rand.Float32() < 0.2 && i > 0 {
if rand.Float32() < 0.2 {
nm.Delete(Uint64ToNeedleId(uint64(rand.Int63n(int64(i))+1)), Uint32ToOffset(uint32(0)))
}
}
+2 -6
View File
@@ -41,9 +41,7 @@ func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
expectedLastUpdateTime += 2000
}
}
@@ -88,9 +86,7 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
expectedLastUpdateTime += 2000
}
}
+5 -6
View File
@@ -2,16 +2,15 @@ package topology
import (
"errors"
"math/rand"
"strings"
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"math/rand"
"strings"
"sync"
"sync/atomic"
)
type NodeId string
@@ -74,7 +73,7 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption
n.RUnlock()
if len(candidates) < numberOfNodes {
glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
return nil, nil, errors.New("Not enough data nodes found!")
return nil, nil, errors.New("No enough data node found!")
}
//pick nodes randomly by weights, the node picked earlier has higher final weights
+13 -19
View File
@@ -11,8 +11,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
backoff "github.com/cenkalti/backoff/v4"
hashicorpRaft "github.com/hashicorp/raft"
"github.com/seaweedfs/raft"
@@ -37,7 +35,7 @@ type Topology struct {
volumeSizeLimit uint64
replicationAsMin bool
isDisableVacuum bool
isDisableVacuum bool
Sequence sequence.Sequencer
@@ -98,23 +96,19 @@ func (t *Topology) IsLeader() bool {
}
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
exponentialBackoff := backoff.NewExponentialBackOff()
exponentialBackoff.InitialInterval = 100 * time.Millisecond
exponentialBackoff.MaxElapsedTime = 20 * time.Second
leaderNotSelected := errors.New("leader not selected yet")
l, err = backoff.RetryWithData(
func() (l pb.ServerAddress, err error) {
l, err = t.MaybeLeader()
if err == nil && l == "" {
err = leaderNotSelected
}
return l, err
},
exponentialBackoff)
if err == leaderNotSelected {
l = ""
for count := 0; count < 3; count++ {
l, err = t.MaybeLeader()
if err != nil {
return
}
if l != "" {
break
}
time.Sleep(time.Duration(5+count) * time.Second)
}
return l, err
return
}
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
-25
View File
@@ -395,31 +395,6 @@ func (vl *VolumeLayout) setVolumeWritable(vid needle.VolumeId) bool {
return true
}
func (vl *VolumeLayout) SetVolumeReadOnly(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
if _, ok := vl.vid2location[vid]; ok {
vl.readonlyVolumes.Add(vid, dn)
return vl.removeFromWritable(vid)
}
return true
}
func (vl *VolumeLayout) SetVolumeWritable(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
if _, ok := vl.vid2location[vid]; ok {
vl.readonlyVolumes.Remove(vid, dn)
}
if vl.enoughCopies(vid) {
return vl.setVolumeWritable(vid)
}
return false
}
func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId) bool {
vl.accessLock.Lock()
defer vl.accessLock.Unlock()
+1 -1
View File
@@ -5,7 +5,7 @@ import (
)
var (
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.44)
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.42)
VERSION = sizeLimit + " " + VERSION_NUMBER
COMMIT = ""
)