Compare commits

...
Author SHA1 Message Date
chrislu c057ae6004 enable gocdk in the generated local image
fix https://github.com/seaweedfs/seaweedfs/issues/4256
2023-02-27 19:19:12 -08:00
chrislu 5e42b69841 add uint64 2023-02-16 21:34:23 -08:00
chrislu 59f44b70c3 tests can compile 2023-02-13 00:01:14 -08:00
chrislu dc4ed2cd9b do not move cloud tier volumes
fix https://github.com/seaweedfs/seaweedfs/issues/4195
2023-02-09 22:17:54 -08:00
chrislu 31bb91583f fix bug when vid not found
fix https://github.com/seaweedfs/seaweedfs/issues/4193
2023-02-09 17:30:44 -08:00
chrislu 532a352121 update goexif version 2023-02-09 14:53:19 -08:00
Chris Lu bac45912e3 go 1.20 (#4192) 2023-02-08 16:41:23 -08:00
chrislu 79d68a81fd text change 2023-02-07 15:00:41 -08:00
chrislu 999a126a3f typo 2023-02-07 14:59:47 -08:00
chrislu 1c2364936a detect invalid bucket name
fix https://github.com/seaweedfs/seaweedfs/issues/4143
2023-02-07 14:57:29 -08:00
chrislu 5de93fe442 refactoring 2023-02-07 14:45:20 -08:00
Eric Yang 5083429704 ADHOC: s3 bucket name verification (#4189)
* ADHOC: s3 bucket name verification

* add test
2023-02-07 14:37:29 -08:00
Neo d5f77706a8 volume:fix return if Replicate write error (#4188) 2023-02-06 22:34:56 -08:00
dependabot[bot] 39be630242 build(deps): bump github.com/schollz/progressbar/v3 from 3.12.2 to 3.13.0 (#4185)
build(deps): bump github.com/schollz/progressbar/v3

Bumps [github.com/schollz/progressbar/v3](https://github.com/schollz/progressbar) from 3.12.2 to 3.13.0.
- [Release notes](https://github.com/schollz/progressbar/releases)
- [Commits](https://github.com/schollz/progressbar/compare/v3.12.2...v3.13.0)

---
updated-dependencies:
- dependency-name: github.com/schollz/progressbar/v3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:32:00 -08:00
dependabot[bot] 3189ee9e57 build(deps): bump docker/setup-buildx-action from 2.4.0 to 2.4.1 (#4181)
Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.4.0 to 2.4.1.
- [Release notes](https://github.com/docker/setup-buildx-action/releases)
- [Commits](https://github.com/docker/setup-buildx-action/compare/15c905b16b06416d2086efa066dd8e3a35cc7f98...f03ac48505955848960e80bbb68046aa35c7b9e7)

---
updated-dependencies:
- dependency-name: docker/setup-buildx-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:30:55 -08:00
dependabot[bot] fd517c7a6e build(deps): bump docker/build-push-action from 3.3.0 to 4.0.0 (#4182)
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3.3.0 to 4.0.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](https://github.com/docker/build-push-action/compare/37abcedcc1da61a57767b7588cb9d03eb57e28b3...3b5e8027fcad23fda98b2e3ac259d8d67585f671)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:30:44 -08:00
dependabot[bot] e19ca6ac3c build(deps): bump github.com/klauspost/reedsolomon from 1.11.5 to 1.11.6 (#4183)
Bumps [github.com/klauspost/reedsolomon](https://github.com/klauspost/reedsolomon) from 1.11.5 to 1.11.6.
- [Release notes](https://github.com/klauspost/reedsolomon/releases)
- [Commits](https://github.com/klauspost/reedsolomon/compare/v1.11.5...v1.11.6)

---
updated-dependencies:
- dependency-name: github.com/klauspost/reedsolomon
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:30:34 -08:00
dependabot[bot] 89cee70c2e build(deps): bump github.com/Jille/raft-grpc-transport from 1.3.0 to 1.4.0 (#4184)
build(deps): bump github.com/Jille/raft-grpc-transport

Bumps [github.com/Jille/raft-grpc-transport](https://github.com/Jille/raft-grpc-transport) from 1.3.0 to 1.4.0.
- [Release notes](https://github.com/Jille/raft-grpc-transport/releases)
- [Commits](https://github.com/Jille/raft-grpc-transport/compare/v1.3.0...v1.4.0)

---
updated-dependencies:
- dependency-name: github.com/Jille/raft-grpc-transport
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:30:22 -08:00
dependabot[bot] c19d121e5c build(deps): bump github.com/tikv/client-go/v2 from 2.0.4 to 2.0.5 (#4186)
Bumps [github.com/tikv/client-go/v2](https://github.com/tikv/client-go) from 2.0.4 to 2.0.5.
- [Release notes](https://github.com/tikv/client-go/releases)
- [Commits](https://github.com/tikv/client-go/compare/v2.0.4...v2.0.5)

---
updated-dependencies:
- dependency-name: github.com/tikv/client-go/v2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:30:07 -08:00
dependabot[bot] 0ce9704c3d build(deps): bump go.etcd.io/etcd/client/v3 from 3.5.6 to 3.5.7 (#4187)
Bumps [go.etcd.io/etcd/client/v3](https://github.com/etcd-io/etcd) from 3.5.6 to 3.5.7.
- [Release notes](https://github.com/etcd-io/etcd/releases)
- [Changelog](https://github.com/etcd-io/etcd/blob/main/Dockerfile-release.ppc64le)
- [Commits](https://github.com/etcd-io/etcd/compare/v3.5.6...v3.5.7)

---
updated-dependencies:
- dependency-name: go.etcd.io/etcd/client/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-02-06 08:29:54 -08:00
Tobias Gurtzick 4cd6f64d92 Update backers.md (#4179) 2023-02-06 01:08:09 -08:00
29 changed files with 1924 additions and 62 deletions

No files matched your search

+2 -2
View File
@@ -36,7 +36,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
with:
buildkitd-flags: "--debug"
-
@@ -56,7 +56,7 @@ jobs:
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
with:
buildkitd-flags: "--debug"
-
@@ -57,7 +57,7 @@ jobs:
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -48,7 +48,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # v1
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
@@ -47,7 +47,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@37abcedcc1da61a57767b7588cb9d03eb57e28b3 # v2
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+1
View File
@@ -7,6 +7,7 @@
- [Evercam Camera Management Software](https://evercam.io/)
- [Spherical Elephant GmbH](https://www.sphericalelephant.com)
- [WizardTales GmbH](https://www.wizardtales.com)
- <h2 align="center">Backers</h2>
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.19-alpine as builder
FROM golang:1.20-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.19-buster as builder
FROM golang:1.20-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.19-buster as builder
FROM golang:1.20-buster as builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+2 -1
View File
@@ -62,7 +62,8 @@ k8s: build
dev_registry: build
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate: build
dev_replicate:
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
+14 -13
View File
@@ -1,6 +1,6 @@
module github.com/seaweedfs/seaweedfs
go 1.19
go 1.20
require (
cloud.google.com/go v0.107.0 // indirect
@@ -58,14 +58,14 @@ require (
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/klauspost/reedsolomon v1.11.5
github.com/klauspost/reedsolomon v1.11.6
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.7
github.com/linxGnu/grocksdb v1.7.14
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -82,7 +82,7 @@ require (
github.com/prometheus/procfs v0.9.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.3 // indirect
@@ -105,7 +105,7 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.6
go.etcd.io/etcd/client/v3 v3.5.7
go.mongodb.org/mongo-driver v1.11.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.28.0
@@ -139,7 +139,7 @@ require (
)
require (
github.com/Jille/raft-grpc-transport v1.3.0
github.com/Jille/raft-grpc-transport v1.4.0
github.com/arangodb/go-driver v1.4.1
github.com/armon/go-metrics v0.4.1
github.com/fluent/fluent-logger-golang v1.9.0
@@ -148,8 +148,8 @@ require (
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/rabbitmq/amqp091-go v1.5.0
github.com/schollz/progressbar/v3 v3.12.2
github.com/tikv/client-go/v2 v2.0.4
github.com/schollz/progressbar/v3 v3.13.0
github.com/tikv/client-go/v2 v2.0.5
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
@@ -209,15 +209,16 @@ require (
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f // indirect
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
@@ -225,8 +226,8 @@ require (
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.24.0 // indirect
+37 -28
View File
@@ -47,6 +47,7 @@ cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp
cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE=
cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw=
cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY=
cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg=
cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI=
cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4=
cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk=
@@ -89,6 +90,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA=
cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw=
cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc=
cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY=
cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s=
cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI=
@@ -155,6 +157,7 @@ cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4c
cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM=
cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo=
cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ=
cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g=
@@ -176,6 +179,7 @@ cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1
cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg=
cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk=
cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w=
cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU=
cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI=
cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8=
cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc=
@@ -228,6 +232,7 @@ cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiP
cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8=
cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8=
cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08=
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
@@ -285,9 +290,12 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
cloud.google.com/go/pubsub v1.24.0/go.mod h1:rWv09Te1SsRpRGPiWOMDKraMQTJyJps4MkUCoMGUgqw=
cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
cloud.google.com/go/pubsub v1.27.0/go.mod h1:BgkDyjrFNV8c7txDxPrlQkM/XtbJQVEeAWmt56lVVf8=
cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
cloud.google.com/go/pubsub v1.28.0 h1:XzabfdPx/+eNrsVVGLFgeUnQQKPGkMb8klRCeYK52is=
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o=
cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk=
@@ -343,6 +351,7 @@ cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5
cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU=
cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4=
cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw=
cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos=
cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM=
cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ=
cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0=
@@ -484,8 +493,8 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/GoogleCloudPlatform/cloudsql-proxy v1.31.2/go.mod h1:qR6jVnZTKDCW3j+fC9mOEPHm++1nKDMkqbbkD6KNsfo=
github.com/GoogleCloudPlatform/cloudsql-proxy v1.33.1/go.mod h1:n3KDPrdaY2p9Nr0B1allAdjYArwIpXQcitNbsS/Qiok=
github.com/Jille/raft-grpc-transport v1.3.0 h1:s6a7F4k23n7IlH5O6fWW5xz2sxbz6RIB4CzWlsvmbpY=
github.com/Jille/raft-grpc-transport v1.3.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ=
github.com/Jille/raft-grpc-transport v1.4.0 h1:Kwk+IceQD8MpLKOulBu2ignX+aZAEjOhffEhN44sdzQ=
github.com/Jille/raft-grpc-transport v1.4.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -534,7 +543,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -1280,7 +1288,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.2/go.mod h1:chrfS3YoLAlKTRE5cFWvCbt8uGAjshktT4PveTUpsFQ=
@@ -1502,8 +1509,8 @@ github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UH
github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0=
github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/klauspost/reedsolomon v1.11.5 h1:8ebqrZbby2dplht2gUPplizNlvYGCghRRfq5F9SFYKM=
github.com/klauspost/reedsolomon v1.11.5/go.mod h1:lbYSjK96uTl22BD+PzgzlsCal+PW/0yLKlQP4MMT+i4=
github.com/klauspost/reedsolomon v1.11.6 h1:h0MUpEzmretucmlelC3EefQHKgk6vWpKz/ctB/tmaEs=
github.com/klauspost/reedsolomon v1.11.6/go.mod h1:cuXqklb3LNaurR5MVjy7WLXAEUqGz4I0Uc+rnQ7POUg=
github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1582,8 +1589,9 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
@@ -1788,11 +1796,10 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 h1:ZiCJcEzmmF5xNgt8GIXekd3WQXI/22kzYQnrHi3Fc/4=
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125/go.mod h1:+on3Lfk/fb1lXkud3XvskJumhSIEEgN2TTbMObUlrxE=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
@@ -1826,7 +1833,6 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
@@ -1919,11 +1925,13 @@ github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiB
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
github.com/schollz/progressbar/v3 v3.12.2 h1:yLqqqpQNMxGxHY8uEshRihaHWwa0rf0yb7/Zrpgq2C0=
github.com/schollz/progressbar/v3 v3.12.2/go.mod h1:HFJYIYQQJX32UJdyoigUl19xoV6aMwZt6iX/C30RWfg=
github.com/schollz/progressbar/v3 v3.13.0 h1:9TeeWRcjW2qd05I8Kf9knPkW4vLM/hYoa6z9ABvxje8=
github.com/schollz/progressbar/v3 v3.13.0/go.mod h1:ZBYnSuLAX2LU8P8UiKN/KgF2DY58AJC8yfVYLPC8Ly4=
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps=
github.com/seaweedfs/goexif v2.0.0+incompatible/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
github.com/seaweedfs/raft v1.1.0 h1:Oy1mf3MzktDzNyXamD5lAZirLjEqPS7FzZoxLY0i8SU=
@@ -2032,10 +2040,12 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2Xxo=
github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tikv/client-go/v2 v2.0.5 h1:GEWZqzVMmSCrQoWVcB/6234n3/hYfpZkJldnZnz5wYI=
github.com/tikv/client-go/v2 v2.0.5/go.mod h1:DtwnMX8PDLcbXn2T4AyiCFPjmzTr1F4MQzJQpQhJeLM=
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f h1:iY/RztOIZ2nTbINUiLGsSv3SUGoEiub1GN0SKVKHJYg=
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f/go.mod h1:jb9oq6rN4U0U3FZdvqWlpi9rZzFJxiOlvZ3aj5BTpg8=
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f h1:xHPPej9Z8IrYdyLED4byOGtGm/7yhxyRUf4m93CpDyg=
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f/go.mod h1:0fdKmj9cafPLsHAeVAcgB50Uj018WGfMC1tgvzWXXQQ=
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -2128,15 +2138,15 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A=
go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8=
go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY=
go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA=
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU=
go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ=
go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg=
go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY=
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E=
go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk=
go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4=
go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw=
go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
@@ -2367,7 +2377,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -2867,7 +2876,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -2983,9 +2991,11 @@ google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnp
google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s=
google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
@@ -3021,7 +3031,6 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+6
View File
@@ -0,0 +1,6 @@
all: gen
.PHONY : gen
gen:
protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint16 struct {
}
const SIZE_Uint16 = 2
func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
return binary.BigEndian.Uint16(buf)
}
return 0
}
func WriteUint16s(buf []byte, data []uint16) (err error) {
off := 0
size := len(data)
if len(buf) < size<<1 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint16(buf[off:], dat)
off += SIZE_Uint16
}
return nil
}
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint32 struct {
}
const SIZE_Uint32 = 4
func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
return binary.BigEndian.Uint32(buf)
}
return 0
}
func WriteUint32s(buf []byte, data []uint32) (err error) {
off := 0
size := len(data)
if len(buf) < size<<2 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint32(buf[off:], dat)
off += SIZE_Uint32
}
return nil
}
+32
View File
@@ -0,0 +1,32 @@
package data
import (
"encoding/binary"
"fmt"
"io"
)
type ColumnUint64 struct {
}
const SIZE_Uint64 = 8
func (c *ColumnUint64) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint64 {
if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
return binary.BigEndian.Uint64(buf)
}
return 0
}
func WriteUint64s(buf []byte, data []uint64) (err error) {
off := 0
size := len(data)
if len(buf) < size<<3 {
return fmt.Errorf("buf too small")
}
for _, dat := range data {
binary.BigEndian.PutUint64(buf[off:], dat)
off += SIZE_Uint64
}
return nil
}
+103
View File
@@ -0,0 +1,103 @@
syntax = "proto3";
package columnar_pb;
option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb";
message FileId {
uint32 volume_id = 1;
uint64 file_key = 2;
fixed32 cookie = 3;
}
enum LogicalType {
Uint8 = 0;
Uint16 = 1;
Float32 = 4;
}
message ColumnUint16 {
uint32 base = 1;
uint32 min = 3;
uint32 max = 4;
}
message ColumnUint32 {
uint32 base = 1;
uint32 min = 3;
uint32 max = 4;
}
message ColumnFloat32 {
uint32 min = 3;
uint32 max = 4;
}
message ColumnSplit {
// The ids of the fields/columns in this file
int32 field_id = 1;
FileId file_id = 2;
int64 row_offset = 3;
int32 row_count = 4;
oneof storage_type {
ColumnUint16 meta_uint16 = 8;
ColumnUint32 meta_uint32 = 9;
ColumnFloat32 meta_float32 = 10;
}
}
message Snapshot {
// All fields of the dataset, including the nested fields.
repeated Field fields = 1;
repeated string data_files = 2;
// Snapshot version number.
uint64 version = 3;
}
message DataFile {
repeated int32 field_ids = 1;
repeated RowGroup row_groups = 2;
}
message RowGroup {
int64 row_offset = 1;
int32 row_count = 2;
repeated ColumnSplit column_splits = 3;
}
// Field metadata for a column.
message Field {
enum Type {
PARENT = 0;
REPEATED = 1;
LEAF = 2;
}
Type type = 1;
// Fully qualified name.
string name = 2;
/// Field Id.
int32 id = 3;
/// Parent Field ID. If not set, this is a top-level column.
int32 parent_id = 4;
// Logical types, support parameterized Arrow Type.
LogicalType logical_type = 5;
// If this field is nullable.
bool nullable = 6;
}
message AnyValue {
oneof value {
bytes bytes_value = 1;
bool bool_value = 2;
uint64 int64_value = 3;
uint32 int32_value = 4;
double double_value = 5;
}
}
File diff suppressed because it is too large. Load diff
+91
View File
@@ -0,0 +1,91 @@
package data
import "fmt"
type Datum interface {
Compare(other Datum) (int, error)
}
type Datums []Datum
type DUint16 uint16
type DUint32 uint32
type DUint64 uint64
type dNull struct{}
var (
DNull Datum = dNull{}
)
func (d dNull) Compare(other Datum) (int, error) {
if other == DNull {
return 0, nil
}
return -1, nil
}
func NewDUint16(d DUint16) *DUint16 {
return &d
}
func NewDUint32(d DUint32) *DUint32 {
return &d
}
func NewDUint64(d DUint64) *DUint64 {
return &d
}
func (d *DUint16) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint16
switch t := other.(type) {
case *DUint16:
otherV = *t
default:
return 0, fmt.Errorf("unsupported")
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
func (d *DUint32) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint32
switch t := other.(type) {
case *DUint32:
otherV = *t
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
func (d *DUint64) Compare(other Datum) (int, error) {
if other == DNull {
return 1, nil
}
thisV := *d
var otherV DUint64
switch t := other.(type) {
case *DUint64:
otherV = *t
}
if thisV < otherV {
return -1, nil
}
if thisV > otherV {
return 1, nil
}
return 0, nil
}
+259
View File
@@ -0,0 +1,259 @@
package data
import (
"encoding/binary"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"testing"
)
func TestRead(t *testing.T) {
x := make([]uint16, 128)
y := make([]uint32, 128)
z := make([]uint64, 128)
for i := range x {
x[i] = uint16(i)
}
for i := range y {
y[i] = uint32(i * 2)
}
for i := range z {
z[i] = uint64(i * 3)
}
xbuf := make([]byte, len(x)*SIZE_Uint16)
ybuf := make([]byte, len(x)*SIZE_Uint32)
zbuf := make([]byte, len(x)*SIZE_Uint64)
WriteUint16s(xbuf, x)
WriteUint32s(ybuf, y)
WriteUint64s(zbuf, z)
df := &DataFile{
xbuf: xbuf,
ybuf: ybuf,
zbuf: zbuf,
xLen: len(xbuf),
yLen: len(ybuf),
zLen: len(zbuf),
xReaderAt: util.NewBytesReader(xbuf),
yReaderAt: util.NewBytesReader(ybuf),
zReaderAt: util.NewBytesReader(zbuf),
}
dataLayout := make(map[FieldName]DataLayout)
dataLayout["x"] = DataLayout{
LayoutType: Uint16,
SortType: Unsorted,
}
dataLayout["y"] = DataLayout{
LayoutType: Uint32,
SortType: Unsorted,
}
dataLayout["z"] = DataLayout{
LayoutType: Uint64,
SortType: Unsorted,
}
rows, err := df.ReadRows("x", dataLayout, Equal, NewDUint16(65))
if err != nil {
fmt.Printf("err: %v", err)
return
}
for _, row := range rows {
fmt.Printf("row %d width %d ", row.index, len(row.Datums))
for i, d := range row.Datums {
fmt.Printf("%d: %v ", i, d)
}
fmt.Println()
}
}
type Operator int32
type LayoutType int32
type SortType int32
const (
Equal Operator = 0
GreaterThan
GreaterOrEqual
LessThan
LessOrEqual
Uint16 LayoutType = 0
Uint32 = 1
Uint64 = 2
Unsorted SortType = 0
Ascending
Descending
)
type DataFile struct {
xbuf []byte
ybuf []byte
zbuf []byte
xReaderAt io.ReaderAt
yReaderAt io.ReaderAt
zReaderAt io.ReaderAt
xLen int
yLen int
zLen int
}
type DataLayout struct {
LayoutType
SortType
}
type FieldName string
func (d *DataFile) ReadRows(field FieldName, layout map[FieldName]DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
if field == "x" {
rows, err = pushDownReadRows(d.xReaderAt, d.xLen, layout[field], op, operand)
if err != nil {
return
}
err = hydrateRows(d.yReaderAt, d.yLen, layout["y"], rows)
if err != nil {
return
}
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
if err != nil {
return
}
}
if field == "y" {
rows, err = pushDownReadRows(d.yReaderAt, d.yLen, layout[field], op, operand)
if err != nil {
return
}
err = hydrateRows(d.xReaderAt, d.xLen, layout["x"], rows)
if err != nil {
return
}
err = hydrateRows(d.zReaderAt, d.zLen, layout["z"], rows)
if err != nil {
return
}
}
return
}
type Row struct {
index int
Datums
}
func pushDownReadRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
if layout.LayoutType == Uint16 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint16)
for i := 0; i < dataLen; i += SIZE_Uint16 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint16 && err == nil {
d := NewDUint16(DUint16(binary.BigEndian.Uint16(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(1)
rows = append(rows, &Row{
index: i / SIZE_Uint16,
Datums: []Datum{d},
})
}
}
}
}
}
if layout.LayoutType == Uint32 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint32)
for i := 0; i < dataLen; i += SIZE_Uint32 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint32 && err == nil {
d := NewDUint32(DUint32(binary.BigEndian.Uint32(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(2)
rows = append(rows, &Row{
index: i / SIZE_Uint32,
Datums: []Datum{d},
})
}
}
}
}
}
if layout.LayoutType == Uint64 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint64)
for i := 0; i < dataLen; i += SIZE_Uint64 {
if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint64 && err == nil {
d := NewDUint64(DUint64(binary.BigEndian.Uint64(buf)))
cmp, err := d.Compare(operand)
if err != nil {
return rows, err
}
if cmp == 0 && op == Equal {
println(2)
rows = append(rows, &Row{
index: i / SIZE_Uint64,
Datums: []Datum{d},
})
}
}
}
}
}
return
}
func hydrateRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, rows []*Row) (err error) {
if layout.LayoutType == Uint16 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint16)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
t := binary.BigEndian.Uint16(buf)
d := NewDUint16(DUint16(t))
println(3, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
if layout.LayoutType == Uint32 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint32)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
t := binary.BigEndian.Uint32(buf)
d := NewDUint32(DUint32(t))
println(4, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
if layout.LayoutType == Uint64 {
if layout.SortType == Unsorted {
buf := make([]byte, SIZE_Uint64)
for _, row := range rows {
if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint64); n == SIZE_Uint64 && err == nil {
t := binary.BigEndian.Uint64(buf)
d := NewDUint64(DUint64(t))
println(5, "add", t)
row.Datums = append(row.Datums, d)
}
}
}
}
return
}
+9
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"net/http"
@@ -84,6 +85,14 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketHandler %s", bucket)
// validate the bucket name
err := s3bucket.VerifyS3BucketName(bucket)
if err != nil {
glog.Errorf("put invalid bucket name: %v %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName)
return
}
// avoid duplicated buckets
errCode := s3err.ErrNone
if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+40
View File
@@ -0,0 +1,40 @@
package s3bucket
import (
"fmt"
"net"
"strings"
"unicode"
)
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
func VerifyS3BucketName(name string) (err error) {
if len(name) < 3 || len(name) > 63 {
return fmt.Errorf("bucket name must between [3, 63] characters")
}
for idx, ch := range name {
if !(unicode.IsLower(ch) || ch == '.' || ch == '-' || unicode.IsNumber(ch)) {
return fmt.Errorf("bucket name can only contain lower case characters, numbers, dots, and hyphens")
}
if idx > 0 && (ch == '.' && name[idx-1] == '.') {
return fmt.Errorf("bucket names must not contain two adjacent periods")
}
//TODO buckets with s3 transfer acceleration cannot have . in name
}
if name[0] == '.' || name[0] == '-' {
return fmt.Errorf("name must start with number or lower case character")
}
if name[len(name)-1] == '.' || name[len(name)-1] == '-' {
return fmt.Errorf("name must end with number or lower case character")
}
if strings.HasPrefix(name, "xn--") {
return fmt.Errorf("prefix xn-- is reserved and not allowed in bucket prefix")
}
if strings.HasSuffix(name, "-s3alias") {
return fmt.Errorf("suffix -s3alias is reserved and not allowed in bucket suffix")
}
if net.ParseIP(name) != nil {
return fmt.Errorf("bucket name cannot be ip addresses")
}
return nil
}
+37
View File
@@ -0,0 +1,37 @@
package s3bucket
import (
"github.com/stretchr/testify/assert"
"testing"
)
func Test_verifyBucketName(t *testing.T) {
invalidS3BucketNames := []string{
"A9325325b",
"123.12.153.10",
"abc214..2",
"d",
".ewfs3253543",
"grehtrry-",
"----------",
"x@fdsgr032",
}
for _, invalidName := range invalidS3BucketNames {
err := VerifyS3BucketName(invalidName)
assert.NotNil(t, err)
}
validS3BucketName := []string{
"a9325325b",
"999.12.153.10",
"abc214.2",
"3d3d3d",
"ewfs3253543",
"grehtrry-a",
"0----------0",
"xafdsgr032",
}
for _, invalidName := range validS3BucketName {
err := VerifyS3BucketName(invalidName)
assert.Nil(t, err)
}
}
@@ -48,6 +48,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
isUnchanged, writeError := topology.ReplicatedWrite(vs.GetMaster, vs.grpcDialOption, vs.store, volumeId, reqNeedle, r, contentMd5)
if writeError != nil {
writeJsonError(w, r, http.StatusInternalServerError, writeError)
return
}
// http 204 status code does not allow body
+1 -1
View File
@@ -99,7 +99,7 @@ func doEcEncode(commandEnv *CommandEnv, collection string, vid needle.VolumeId,
// find volume location
locations, found := commandEnv.MasterClient.GetLocations(uint32(vid))
if !found && len(locations) > 0 {
if !found {
return fmt.Errorf("volume %d not found", vid)
}
+7 -2
View File
@@ -4,11 +4,11 @@ import (
"context"
"flag"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
"io"
"os"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func init() {
@@ -42,6 +42,11 @@ func (c *commandS3BucketCreate) Do(args []string, commandEnv *CommandEnv, writer
return fmt.Errorf("empty bucket name")
}
err = s3bucket.VerifyS3BucketName(*bucketName)
if err != nil {
return err
}
err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+4
View File
@@ -329,6 +329,10 @@ func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*Vol
return false, fmt.Errorf("lock is lost")
}
if candidateVolume.RemoteStorageName != "" {
return false, nil
}
if candidateVolume.ReplicaPlacement > 0 {
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(candidateVolume.ReplicaPlacement))
if !isGoodMove(replicaPlacement, volumeReplicas[candidateVolume.Id], fullNode, emptyNode) {