Compare commits

..
Author SHA1 Message Date
Chris LuandClaude Opus 5.5 0662a982ba ci: install Caddy from the pinned GitHub release
caddyserver.com/api/download now answers 401 without a login or API
key, so the awscli-over-https job failed in Setup Caddy on every run.
Download the v2.11.9 release tarball instead and verify it against the
release's SHA-512 checksums.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 05:40:06 -08:00
57 changed files with 165 additions and 1934 deletions

No files matched your search

@@ -56,8 +56,12 @@ jobs:
- name: Setup Caddy
run: |
curl -fsSL "https://caddyserver.com/api/download?os=linux&arch=amd64" -o caddy
chmod +x caddy
CADDY_VERSION=2.11.9
base="https://github.com/caddyserver/caddy/releases/download/v${CADDY_VERSION}"
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_linux_amd64.tar.gz"
curl -fsSLO "${base}/caddy_${CADDY_VERSION}_checksums.txt"
grep " caddy_${CADDY_VERSION}_linux_amd64.tar.gz$" "caddy_${CADDY_VERSION}_checksums.txt" | sha512sum -c -
tar -xzf "caddy_${CADDY_VERSION}_linux_amd64.tar.gz" caddy
./caddy version
echo "{
auto_https disable_redirects
+3 -3
View File
@@ -2,6 +2,7 @@ package command
import (
"context"
"crypto/sha1"
"crypto/tls"
"fmt"
"io/fs"
@@ -448,9 +449,8 @@ func copyDirForReadOnly(src string) (string, error) {
if err != nil {
abs = src
}
h := util.NewSha1()
h.Write([]byte(abs))
dst := filepath.Join(os.TempDir(), fmt.Sprintf("weed-master-ro-%x", h.Sum(nil)[:8]))
sum := sha1.Sum([]byte(abs))
dst := filepath.Join(os.TempDir(), fmt.Sprintf("weed-master-ro-%x", sum[:8]))
if fi, err := os.Stat(dst); err == nil && fi.IsDir() &&
fi.Mode().Perm() == 0700 && util.DirOwnedBySelf(fi) {
if !newestMtime(dst).Before(newestMtime(src)) {
-4
View File
@@ -6,7 +6,6 @@ import (
"bytes"
"compress/gzip"
"context"
"crypto/fips140"
"crypto/md5"
"encoding/hex"
"encoding/json"
@@ -153,9 +152,6 @@ func downloadRelease(ctx context.Context, target string, ver string) (version st
}
suffix := fmt.Sprintf("%s_%s%s%s.%s", runtime.GOOS, runtime.GOARCH, fullSuffix, largeDiskSuffix, ext)
if fips140.Enforced() {
return "", fmt.Errorf("the download is checked with MD5, which GODEBUG=fips140=only refuses")
}
md5Filename := fmt.Sprintf("%s.md5", suffix)
_, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename)
if err != nil {
@@ -3,6 +3,7 @@ package filer_etc
import (
"bytes"
"context"
"crypto/md5"
"encoding/json"
"errors"
"fmt"
@@ -14,7 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
@@ -546,7 +546,7 @@ func (store *FilerEtcStore) createIfAbsent(ctx context.Context, subDir, name str
if err != nil {
return err
}
contentMd5 := util.Md5(data)
contentMd5 := md5.Sum(data)
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: filer.IamConfigDirectory + "/" + subDir,
@@ -558,7 +558,7 @@ func (store *FilerEtcStore) createIfAbsent(ctx context.Context, subDir, name str
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(data)),
Md5: contentMd5,
Md5: contentMd5[:],
},
},
OExcl: true,
+2 -1
View File
@@ -2,6 +2,7 @@ package arangodb
import (
"context"
"crypto/md5"
"encoding/binary"
"encoding/hex"
"io"
@@ -13,7 +14,7 @@ import (
// convert a string into arango-key safe hex bytes hash
func hashString(dir string) string {
h := util.NewMd5()
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
return hex.EncodeToString(b)
-37
View File
@@ -99,10 +99,6 @@ type Filer struct {
// aggregator across a replayed peer change's check-and-apply, so the two
// cannot interleave on the same path.
EntryLockTable *util.LockTable[util.FullPath]
// directoryCreateLock covers a directory row's re-check and insert, so a
// missing parent filled in by a nested create cannot replace a directory
// created at the same time.
directoryCreateLock *util.LockTable[util.FullPath]
// remoteTombstonesDone, when non-nil, is closed once the startup tombstone
// rebuild finishes; lazy remote reads wait on it so a pending delete
// cannot resurrect in the gap.
@@ -158,7 +154,6 @@ func NewFiler(masters pb.ServerDiscovery, dialOptions *DialOptions, filerHost pb
persistedLogCache: newPersistedLogCache(persistedLogCacheMaxBytes),
remoteTombstones: newRemoteDeletionTombstones(),
EntryLockTable: util.NewLockTable[util.FullPath](),
directoryCreateLock: util.NewLockTable[util.FullPath](),
}
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
if f.UniqueFilerId < 0 {
@@ -531,7 +526,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
entry.Attr.Atime = entryInitialAtime(entry.Attr)
}
unlockDirectory := func() {}
oldEntry := existing
if oldEntry == nil {
var findErr error
@@ -548,13 +542,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
// itself failed; proceeding would upsert over it.
return fmt.Errorf("find entry %s: %w", entry.FullPath, findErr)
}
if oldEntry == nil && entry.IsDirectory() {
unlockDirectory = f.lockDirectoryCreate(ctx, entry.FullPath)
defer unlockDirectory()
if oldEntry, findErr = f.FindEntry(context.WithValue(ctx, lazyFetchContextKey{}, true), entry.FullPath); findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
return fmt.Errorf("find entry %s: %w", entry.FullPath, findErr)
}
}
}
/*
@@ -585,7 +572,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err)
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
}
unlockDirectory()
// Parents go after the entry: one checked first can be taken by the
// empty-folder cleaner before the entry lands, and nothing would look again.
@@ -607,7 +593,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
stats.FilerObjectSizeBytesHistogram.Observe(float64(entry.Size()))
}
} else {
unlockDirectory()
if o_excl {
glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath)
return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
@@ -654,18 +639,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
return nil
}
// lockDirectoryCreate holds the path's directoryCreateLock; the returned
// unlock may be called more than once. A store transaction keeps its
// connection until it commits, and a holder may need that connection, so
// writes inside one do not take the lock.
func (f *Filer) lockDirectoryCreate(ctx context.Context, p util.FullPath) (unlock func()) {
if f.directoryCreateLock == nil || ctx.Value(storeTransactionKey{}) != nil {
return func() {}
}
lock := f.directoryCreateLock.AcquireLock("createDirectory", p, util.ExclusiveLock)
return sync.OnceFunc(func() { f.directoryCreateLock.ReleaseLock(p, lock) })
}
func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
if level == 0 {
@@ -722,18 +695,8 @@ func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, di
}
glog.V(2).InfofCtx(ctx, "create directory: %s %v", dirPath, dirEntry.Mode)
unlock := f.lockDirectoryCreate(ctx, dirEntry.FullPath)
if current, findErr := f.FindEntry(context.WithValue(ctx, lazyFetchContextKey{}, true), dirEntry.FullPath); current != nil || (findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound)) {
unlock()
if current == nil {
return findErr
}
// Created since the lookup above: take it as found.
return f.ensureParentDirectoryEntry(ctx, entry, dirParts, level, isFromOtherCluster)
}
f.stampEntryCommitTsNs(dirEntry)
mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
unlock()
if mkdirErr != nil {
f.releaseEntryCommitTsNs(dirEntry.CommitTsNs)
if fEntry, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound || fEntry == nil {
+4 -3
View File
@@ -3,6 +3,7 @@ package filer
import (
"bytes"
"context"
"crypto/md5"
"errors"
"fmt"
"io"
@@ -374,7 +375,7 @@ func saveFilerConfConditionally(ctx context.Context, client filer_pb.SeaweedFile
return err
}
content := buf.Bytes()
contentMd5 := util.Md5(content)
contentMd5 := md5.Sum(content)
if snap.entry == nil {
err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
@@ -387,7 +388,7 @@ func saveFilerConfConditionally(ctx context.Context, client filer_pb.SeaweedFile
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(content)),
Md5: contentMd5,
Md5: contentMd5[:],
},
Content: content,
},
@@ -422,7 +423,7 @@ func saveFilerConfConditionally(ctx context.Context, client filer_pb.SeaweedFile
entry.Content = content
entry.Attributes.Mtime = time.Now().Unix()
entry.Attributes.FileSize = uint64(len(content))
entry.Attributes.Md5 = contentMd5
entry.Attributes.Md5 = contentMd5[:]
err := filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
Directory: DirectoryEtcSeaweedFS,
Entry: entry,
-119
View File
@@ -1,119 +0,0 @@
package filer
import (
"context"
"errors"
"os"
"runtime"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// missHookStore runs onMiss, outside the store lock, right after the miss-th
// lookup of path that finds nothing.
type missHookStore struct {
*stubFilerStore
path util.FullPath
miss int32
misses atomic.Int32
onMiss func()
}
func (s *missHookStore) FindEntry(ctx context.Context, p util.FullPath) (*Entry, error) {
e, err := s.stubFilerStore.FindEntry(ctx, p)
if p == s.path && errors.Is(err, filer_pb.ErrNotFound) && s.misses.Add(1) == s.miss {
s.onMiss()
}
return e, err
}
func directoryMarker(p util.FullPath) *Entry {
return &Entry{
FullPath: p,
Attr: Attr{Mode: os.ModeDir | 0o755, Mime: s3_constants.FolderMimeType},
Extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
}
}
func assertDirectoryMarker(t *testing.T, store *stubFilerStore, p util.FullPath) {
t.Helper()
dir, err := store.FindEntry(context.Background(), p)
require.NoError(t, err)
assert.True(t, dir.IsDirectory())
assert.Equal(t, s3_constants.FolderMimeType, dir.Attr.Mime, "the implicit parent replaced the directory created meanwhile")
assert.Contains(t, dir.Extended, s3_constants.ExtETagKey)
}
// A directory created while a nested create is filling in that directory as
// a missing parent keeps what it was created with: the parent row is only
// written where none exists.
func TestImplicitParentKeepsConcurrentDirectory(t *testing.T) {
f, stub := newTestFilerWithStubStore(t)
ctx := context.Background()
dirPath := util.FullPath("/buckets/bkt/markers/dir")
store := &missHookStore{stubFilerStore: stub, path: dirPath, miss: 1}
f.Store = NewFilerStoreWrapper(store)
store.onMiss = func() {
require.NoError(t, f.CreateEntry(ctx, directoryMarker(dirPath), nil, false, false, nil, false, f.MaxFilenameLength))
}
child := &Entry{FullPath: dirPath.Child("child"), Attr: Attr{Mode: 0o644}}
require.NoError(t, f.CreateEntry(ctx, child, nil, false, false, nil, false, f.MaxFilenameLength))
assertDirectoryMarker(t, stub, dirPath)
}
// A directory create arriving after the parent's re-check, before its insert,
// waits for that insert and then updates the row rather than being replaced.
func TestDirectoryCreateWaitsForImplicitParentInsert(t *testing.T) {
f, stub := newTestFilerWithStubStore(t)
ctx := context.Background()
dirPath := util.FullPath("/buckets/bkt/markers/dir")
markerDone := make(chan error, 1)
store := &missHookStore{stubFilerStore: stub, path: dirPath, miss: 2}
f.Store = NewFilerStoreWrapper(store)
store.onMiss = func() {
go func() {
markerDone <- f.CreateEntry(ctx, directoryMarker(dirPath), nil, false, false, nil, false, f.MaxFilenameLength)
}()
waitForDirectoryCreateWaiter(t, markerDone)
}
child := &Entry{FullPath: dirPath.Child("child"), Attr: Attr{Mode: 0o644}}
require.NoError(t, f.CreateEntry(ctx, child, nil, false, false, nil, false, f.MaxFilenameLength))
require.GreaterOrEqual(t, store.misses.Load(), int32(2), "the parent was inserted without a re-check")
require.NoError(t, <-markerDone)
assertDirectoryMarker(t, stub, dirPath)
}
// waitForDirectoryCreateWaiter returns once a goroutine waits in
// lockDirectoryCreate, or once the create has finished without waiting.
func waitForDirectoryCreateWaiter(t *testing.T, done chan error) {
t.Helper()
buf := make([]byte, 1<<20)
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
select {
case err := <-done:
done <- err
return
default:
}
for _, g := range strings.Split(string(buf[:runtime.Stack(buf, true)]), "\n\n") {
if strings.Contains(g, "sync.(*Cond).Wait") && strings.Contains(g, "lockDirectoryCreate") {
return
}
}
}
t.Fatal("the directory create neither waited nor finished")
}
+1 -6
View File
@@ -3,7 +3,6 @@ package leveldb
import (
"bytes"
"context"
"crypto/fips140"
"crypto/md5"
"fmt"
"io"
@@ -118,10 +117,6 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
}
store.dbCount = dbCount
if err := leveldb_handle.CheckKeyHash(store.dbs, fips140.Enforced(), store.ReadOnly); err != nil {
store.Shutdown()
return err
}
return
}
@@ -324,7 +319,7 @@ func getNameFromKey(key []byte) string {
// hash directory, and use last byte for partitioning
func hashToBytes(dir string, dbCount int) ([]byte, int) {
h := weed_util.NewMd5()
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
+1 -6
View File
@@ -3,7 +3,6 @@ package leveldb
import (
"bytes"
"context"
"crypto/fips140"
"crypto/md5"
"fmt"
"io"
@@ -139,10 +138,6 @@ func (store *LevelDB3Store) loadDB(name string) (*leveldb_handle.DB, error) {
glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
return nil, dbErr
}
if err := leveldb_handle.CheckKeyHash([]*leveldb_handle.DB{db}, fips140.Enforced(), store.ReadOnly); err != nil {
db.Close()
return nil, err
}
return db, nil
}
@@ -494,7 +489,7 @@ func getNameFromKey(key []byte) string {
// hash directory
func hashToBytes(dir string) []byte {
h := weed_util.NewMd5()
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
return b
-54
View File
@@ -1,54 +0,0 @@
package leveldb_handle
import (
"errors"
"fmt"
"github.com/syndtr/goleveldb/leveldb"
)
var sha256KeyMarker = []byte("\x00seaweedfs.directory_key_hash.sha256")
// CheckKeyHash refuses databases whose directory keys were hashed differently.
// Under GODEBUG=fips140=only directory keys hash with SHA-256 instead of MD5,
// so an empty database opened that way is marked, and a database holding
// keys of the other kind is refused.
func CheckKeyHash(dbs []*DB, sha256Keys, readOnly bool) error {
var unmarked []*DB
for _, db := range dbs {
marked, empty, err := keyHashState(db)
if err != nil {
return fmt.Errorf("%s: %w", db.name, err)
}
switch {
case marked && !sha256Keys:
return fmt.Errorf("%s was created under GODEBUG=fips140=only and can only be opened with it", db.name)
case !marked && sha256Keys && !empty:
return fmt.Errorf("%s hashes directory keys with MD5, which GODEBUG=fips140=only refuses; copy the metadata with fs.meta.save without fips140=only and fs.meta.load into a new store", db.name)
case !marked && sha256Keys:
unmarked = append(unmarked, db)
}
}
if readOnly {
return nil
}
for _, db := range unmarked {
if err := db.Put(sha256KeyMarker, nil, nil); err != nil {
return fmt.Errorf("%s: %w", db.name, err)
}
}
return nil
}
func keyHashState(db *DB) (marked, empty bool, err error) {
if _, err = db.Get(sha256KeyMarker, nil); err == nil {
return true, false, nil
}
if !errors.Is(err, leveldb.ErrNotFound) {
return false, false, err
}
iter := db.NewIterator(nil, nil)
defer iter.Release()
empty = !iter.First()
return false, empty, iter.Error()
}
@@ -1,65 +0,0 @@
package leveldb_handle
import (
"strings"
"testing"
"github.com/syndtr/goleveldb/leveldb/storage"
)
func openMemDBs(t *testing.T, n int) []*DB {
t.Helper()
var dbs []*DB
for i := 0; i < n; i++ {
db, err := Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
dbs = append(dbs, db)
}
return dbs
}
func TestCheckKeyHashMarksEmptyDatabases(t *testing.T) {
dbs := openMemDBs(t, 2)
if err := CheckKeyHash(dbs, true, false); err != nil {
t.Fatalf("empty databases: %v", err)
}
if err := CheckKeyHash(dbs, true, false); err != nil {
t.Fatalf("marked databases: %v", err)
}
err := CheckKeyHash(dbs, false, false)
if err == nil || !strings.Contains(err.Error(), "fips140=only") {
t.Fatalf("marked databases opened without fips140=only: %v", err)
}
}
func TestCheckKeyHashRefusesMd5Keys(t *testing.T) {
dbs := openMemDBs(t, 2)
if err := CheckKeyHash(dbs, false, false); err != nil {
t.Fatalf("empty databases without fips140=only: %v", err)
}
if err := dbs[1].Put([]byte("0123456789abcdefname"), []byte("entry"), nil); err != nil {
t.Fatal(err)
}
if err := CheckKeyHash(dbs, false, false); err != nil {
t.Fatalf("md5 keys without fips140=only: %v", err)
}
if err := CheckKeyHash(dbs, true, false); err == nil || !strings.Contains(err.Error(), "MD5") {
t.Fatalf("md5 keys under fips140=only: %v", err)
}
if _, err := dbs[0].Get(sha256KeyMarker, nil); err == nil {
t.Fatalf("a refused check must not mark the empty database")
}
}
func TestCheckKeyHashReadOnlyLeavesEmptyDatabasesUnmarked(t *testing.T) {
dbs := openMemDBs(t, 1)
if err := CheckKeyHash(dbs, true, true); err != nil {
t.Fatal(err)
}
if err := CheckKeyHash(dbs, false, false); err != nil {
t.Fatalf("read-only check marked the database: %v", err)
}
}
+5 -5
View File
@@ -3,10 +3,10 @@ package filer
import (
"bytes"
"context"
"crypto/md5"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
@@ -54,7 +54,7 @@ func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, di
Name: name,
})
contentMd5 := util.Md5(content)
contentMd5 := md5.Sum(content)
if err == filer_pb.ErrNotFound {
err = filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
@@ -67,7 +67,7 @@ func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, di
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(content)),
Md5: contentMd5,
Md5: contentMd5[:],
},
Content: content,
},
@@ -75,13 +75,13 @@ func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, di
})
} else if err == nil {
entry := resp.Entry
if bytes.Equal(entry.Content, content) && bytes.Equal(entry.GetAttributes().GetMd5(), contentMd5) {
if bytes.Equal(entry.Content, content) && bytes.Equal(entry.GetAttributes().GetMd5(), contentMd5[:]) {
return nil
}
entry.Content = content
entry.Attributes.Mtime = time.Now().Unix()
entry.Attributes.FileSize = uint64(len(content))
entry.Attributes.Md5 = contentMd5
entry.Attributes.Md5 = contentMd5[:]
err = filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
Directory: dir,
Entry: entry,
+1 -1
View File
@@ -321,7 +321,7 @@ func getNameFromKey(key []byte) string {
// hash directory, and use last byte for partitioning
func hashToBytes(dir string) []byte {
h := weed_util.NewMd5()
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
+1 -1
View File
@@ -466,7 +466,7 @@ func runOwnTxn(ctx context.Context, run func() (inContext bool, err error)) erro
// Encoding Functions
func hashToBytes(dir string) []byte {
h := util.NewSha1()
h := sha1.New()
io.WriteString(h, dir)
b := h.Sum(nil)
return b
+2 -2
View File
@@ -2,18 +2,18 @@ package iam
import (
"crypto/rand"
"crypto/sha1"
"fmt"
"math/big"
"sort"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// Hash computes a SHA1 hash of the input string.
func Hash(s *string) string {
h := util.NewSha1()
h := sha1.New()
h.Write([]byte(*s))
return fmt.Sprintf("%x", h.Sum(nil))
}
@@ -29,19 +29,11 @@ func (f *refusingFiler) CreateEntry(context.Context, *filer_pb.CreateEntryReques
}, nil
}
func (f *refusingFiler) ListEntries(*filer_pb.ListEntriesRequest, filer_pb.SeaweedFiler_ListEntriesServer) error {
return nil
}
func startRefusingFiler(t *testing.T) (*refusingFiler, func() string) {
filer := &refusingFiler{}
return filer, startTestFiler(t, filer)
}
func startTestFiler(t *testing.T, filer filer_pb.SeaweedFilerServer) func() string {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
filer := &refusingFiler{}
server := pb.NewGrpcServer()
filer_pb.RegisterSeaweedFilerServer(server, filer)
go func() { _ = server.Serve(lis) }()
@@ -54,7 +46,7 @@ func startTestFiler(t *testing.T, filer filer_pb.SeaweedFilerServer) func() stri
grpcPort, err := strconv.Atoi(port)
require.NoError(t, err)
address := string(pb.NewServerAddress(host, 1, grpcPort))
return func() string { return address }
return filer, func() string { return address }
}
// The IAM stores must not take a refused create for a stored record. A session
+8 -9
View File
@@ -2,6 +2,7 @@ package integration
import (
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
@@ -12,7 +13,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)
@@ -35,11 +35,11 @@ const (
// OIDCProviderAuditEvent is the payload emitted for each lifecycle event.
type OIDCProviderAuditEvent struct {
Type OIDCProviderAuditEventType `json:"type"`
ARN string `json:"arn"`
URL string `json:"url,omitempty"`
Detail map[string]string `json:"detail,omitempty"`
OccurredAt time.Time `json:"occurredAt"`
Type OIDCProviderAuditEventType `json:"type"`
ARN string `json:"arn"`
URL string `json:"url,omitempty"`
Detail map[string]string `json:"detail,omitempty"`
OccurredAt time.Time `json:"occurredAt"`
}
// OIDCProviderAuditSink consumes lifecycle events. Implementations must be
@@ -149,9 +149,8 @@ func (f *FilerAuditSink) Emit(ctx context.Context, event *OIDCProviderAuditEvent
// at the same nano against different ARNs are perfectly plausible
// (audit-via-batch script). The ARN hash gives us an effectively-
// collision-free name without leaking the ARN into the filer path.
arnHash := util.NewSha1()
arnHash.Write([]byte(event.ARN))
name := fmt.Sprintf("%d-%s-%s.json", event.OccurredAt.UnixNano(), event.Type, hex.EncodeToString(arnHash.Sum(nil)[:8]))
arnSum := sha1.Sum([]byte(event.ARN))
name := fmt.Sprintf("%d-%s-%s.json", event.OccurredAt.UnixNano(), event.Type, hex.EncodeToString(arnSum[:8]))
occurredAt := event.OccurredAt.Unix()
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(addr), f.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
+10 -28
View File
@@ -2,6 +2,7 @@ package integration
import (
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
@@ -18,7 +19,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -234,7 +234,6 @@ type FilerOIDCProviderStore struct {
grpcDialOption grpc.DialOption
basePath string
filerAddressProvider func() string
names *storedNameRekey
}
// NewFilerOIDCProviderStore returns a filer-backed store. Default path
@@ -249,17 +248,6 @@ func NewFilerOIDCProviderStore(config map[string]interface{}, filerAddressProvid
store.basePath = strings.TrimSuffix(bp, "/")
}
}
store.names = &storedNameRekey{
dir: store.basePath,
name: store.fileName,
keyOf: func(content []byte) string {
var record OIDCProviderRecord
if json.Unmarshal(content, &record) != nil {
return ""
}
return record.ARN
},
}
glog.V(2).Infof("Initialized FilerOIDCProviderStore with basePath %s", store.basePath)
return store
}
@@ -277,9 +265,8 @@ func (f *FilerOIDCProviderStore) resolveFilerAddress(filerAddress string) string
func (f *FilerOIDCProviderStore) fileName(arn string) string {
// Hash the ARN to a fixed-width filename so we can store any character set
// (including the URL fragments AWS lets through) without filer-path issues.
h := util.NewSha1()
h.Write([]byte(arn))
return hex.EncodeToString(h.Sum(nil)) + ".json"
h := sha1.Sum([]byte(arn))
return hex.EncodeToString(h[:]) + ".json"
}
// StoreProvider persists `record` as JSON.
@@ -297,7 +284,7 @@ func (f *FilerOIDCProviderStore) StoreProvider(ctx context.Context, filerAddress
return fmt.Errorf("marshal OIDC provider: %v", err)
}
return f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: f.basePath,
Entry: &filer_pb.Entry{
@@ -326,7 +313,7 @@ func (f *FilerOIDCProviderStore) GetProviderByARN(ctx context.Context, filerAddr
}
var data []byte
err := f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: f.basePath,
Name: f.fileName(arn),
@@ -399,7 +386,7 @@ func (f *FilerOIDCProviderStore) ListProviders(ctx context.Context, filerAddress
}
var out []*OIDCProviderRecord
err := f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
// Stream-paginate via StartFromFileName so deployments with more
// than 1000 providers don't get a silently truncated list.
const pageSize = 1000
@@ -459,7 +446,7 @@ func (f *FilerOIDCProviderStore) DeleteProvider(ctx context.Context, filerAddres
if filerAddress == "" {
return fmt.Errorf("filer address is required")
}
return f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
Directory: f.basePath,
Name: f.fileName(arn),
@@ -492,7 +479,7 @@ func (f *FilerOIDCProviderStore) UpdateProvider(ctx context.Context, filerAddres
if arn == "" {
return fmt.Errorf("ARN is required")
}
return f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
for attempt := 0; attempt < maxProviderUpdateAttempts; attempt++ {
var entry *filer_pb.Entry
var current *OIDCProviderRecord
@@ -602,13 +589,8 @@ func (f *FilerOIDCProviderStore) deleteProviderEntryIfUnchanged(ctx context.Cont
return true, nil
}
func (f *FilerOIDCProviderStore) withFilerClient(ctx context.Context, filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddress), f.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
if err := f.names.ensure(ctx, filerAddress, client); err != nil {
return err
}
return fn(client)
})
func (f *FilerOIDCProviderStore) withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddress), f.grpcDialOption, fn)
}
// DeriveOIDCProviderARN turns an issuer URL into the canonical IAM ARN AWS uses.
+8 -26
View File
@@ -2,6 +2,7 @@ package integration
import (
"context"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
@@ -14,7 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)
@@ -92,7 +92,6 @@ type FilerSessionRevocationStore struct {
grpcDialOption grpc.DialOption
basePath string
filerAddressProvider func() string
names *storedNameRekey
}
// NewFilerSessionRevocationStore returns a filer-backed blocklist store.
@@ -108,17 +107,6 @@ func NewFilerSessionRevocationStore(config map[string]interface{}, filerAddressP
store.basePath = strings.TrimSuffix(bp, "/")
}
}
store.names = &storedNameRekey{
dir: store.basePath,
name: store.fileName,
keyOf: func(content []byte) string {
var record RevocationEntry
if json.Unmarshal(content, &record) != nil {
return ""
}
return record.JTI
},
}
glog.V(2).Infof("Initialized FilerSessionRevocationStore with basePath %s", store.basePath)
return store
}
@@ -140,9 +128,8 @@ func (f *FilerSessionRevocationStore) resolveFilerAddress(filerAddress string) s
// hex name that's both filesystem-safe and stable, so the lookup-on-revoke
// path still finds the right entry.
func (f *FilerSessionRevocationStore) fileName(jti string) string {
h := util.NewSha1()
h.Write([]byte(jti))
return hex.EncodeToString(h.Sum(nil)) + ".json"
sum := sha1.Sum([]byte(jti))
return hex.EncodeToString(sum[:]) + ".json"
}
func (f *FilerSessionRevocationStore) Revoke(ctx context.Context, filerAddress string, entry *RevocationEntry) error {
@@ -160,7 +147,7 @@ func (f *FilerSessionRevocationStore) Revoke(ctx context.Context, filerAddress s
if err != nil {
return fmt.Errorf("marshal revocation entry: %v", err)
}
return f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
return f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: f.basePath,
Entry: &filer_pb.Entry{
@@ -187,7 +174,7 @@ func (f *FilerSessionRevocationStore) IsRevoked(ctx context.Context, filerAddres
return false, fmt.Errorf("filer address is required")
}
revoked := false
err := f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
Directory: f.basePath,
Name: f.fileName(jti),
@@ -210,7 +197,7 @@ func (f *FilerSessionRevocationStore) Purge(ctx context.Context, filerAddress st
return 0, fmt.Errorf("filer address is required")
}
count := 0
err := f.withFilerClient(ctx, filerAddress, func(client filer_pb.SeaweedFilerClient) error {
err := f.withFilerClient(filerAddress, func(client filer_pb.SeaweedFilerClient) error {
// Stream-paginate the directory: ListEntriesRequest has no built-in
// "page until done" semantics, so we use StartFromFileName to walk
// the directory in chunks and avoid the previous hardcoded 10k cap.
@@ -261,11 +248,6 @@ func (f *FilerSessionRevocationStore) Purge(ctx context.Context, filerAddress st
return count, err
}
func (f *FilerSessionRevocationStore) withFilerClient(ctx context.Context, filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddress), f.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
if err := f.names.ensure(ctx, filerAddress, client); err != nil {
return err
}
return fn(client)
})
func (f *FilerSessionRevocationStore) withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddress), f.grpcDialOption, fn)
}
-159
View File
@@ -1,159 +0,0 @@
package integration
import (
"context"
"errors"
"fmt"
"io"
"sync"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// storedNameRekey moves records to the name this process derives for their
// key. The names hash the key with util.NewSha1, which changes under
// GODEBUG=fips140=only, so records written in the other mode are otherwise
// missed by lookups and deletes. It runs once per filer address.
type storedNameRekey struct {
dir string
keyOf func(content []byte) string
name func(key string) string
mu sync.Mutex
done sync.Map
}
func (r *storedNameRekey) ensure(ctx context.Context, filerAddress string, client filer_pb.SeaweedFilerClient) error {
if _, ok := r.done.Load(filerAddress); ok {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
if _, ok := r.done.Load(filerAddress); ok {
return nil
}
if err := r.run(ctx, client); err != nil {
return fmt.Errorf("rename records in %s: %w", r.dir, err)
}
r.done.Store(filerAddress, true)
return nil
}
func (r *storedNameRekey) run(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
var misnamed []*filer_pb.Entry
err := listStoredEntries(ctx, client, r.dir, func(entry *filer_pb.Entry) {
key := r.keyOf(entry.Content)
if key != "" && entry.Name != r.name(key) {
misnamed = append(misnamed, entry)
}
})
if err != nil {
return err
}
for _, entry := range misnamed {
if err := r.move(ctx, client, entry); err != nil {
return err
}
}
return nil
}
// move renames entry if it is unchanged, under the lock the stores' conditional
// writes of the new name take, so a record another process already moved and
// then deleted is not written back. A record already under the new name wins.
func (r *storedNameRekey) move(ctx context.Context, client filer_pb.SeaweedFilerClient, listed *filer_pb.Entry) error {
found, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{Directory: r.dir, Name: listed.Name})
if errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
if err != nil {
return fmt.Errorf("lookup %s: %w", listed.Name, err)
}
entry := found.Entry
name := r.name(r.keyOf(entry.Content))
if name == entry.Name {
return nil
}
remove := &filer_pb.ObjectMutation{Type: filer_pb.ObjectMutation_DELETE, Directory: r.dir, Name: entry.Name, IsDeleteData: true}
put := &filer_pb.ObjectMutation{
Type: filer_pb.ObjectMutation_PUT,
Directory: r.dir,
Entry: &filer_pb.Entry{Name: name, Attributes: entry.Attributes, Content: entry.Content},
Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS}}},
}
resp, err := r.transact(ctx, client, name, entry, put, remove)
if err == nil && resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED && resp.FailedMutation != nil && *resp.FailedMutation == 0 {
resp, err = r.transact(ctx, client, name, entry, remove)
}
if err != nil {
return fmt.Errorf("rename %s to %s: %w", entry.Name, name, err)
}
switch {
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
glog.V(1).Infof("%s/%s changed before its rename to %s; leaving it", r.dir, entry.Name, name)
case resp.Error != "":
return fmt.Errorf("rename %s to %s: %s", entry.Name, name, resp.Error)
default:
glog.V(0).Infof("renamed %s/%s to %s", r.dir, entry.Name, name)
}
return nil
}
func (r *storedNameRekey) transact(ctx context.Context, client filer_pb.SeaweedFilerClient, name string, source *filer_pb.Entry, mutations ...*filer_pb.ObjectMutation) (*filer_pb.ObjectTransactionResponse, error) {
target := r.dir + "/" + name
resp, err := client.ObjectTransaction(ctx, &filer_pb.ObjectTransactionRequest{
LockKey: target,
RequestId: uuid.NewString(),
RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + target,
ConditionKey: r.dir + "/" + source.Name,
Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: source,
}}},
Mutations: mutations,
})
if status.Code(err) == codes.FailedPrecondition {
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
}
return resp, err
}
func listStoredEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string, fn func(*filer_pb.Entry)) error {
const pageSize = 1000
startFrom := ""
for {
stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
Directory: dir,
Limit: pageSize,
StartFromFileName: startFrom,
})
if err != nil {
return fmt.Errorf("list %s: %w", dir, err)
}
pageCount := 0
for {
resp, err := stream.Recv()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return fmt.Errorf("list %s: %w", dir, err)
}
if resp.Entry == nil {
continue
}
pageCount++
startFrom = resp.Entry.Name
if !resp.Entry.IsDirectory {
fn(resp.Entry)
}
}
if pageCount < pageSize {
return nil
}
}
}
@@ -1,233 +0,0 @@
package integration
import (
"context"
"crypto/fips140"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"sort"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
// memFiler holds one directory level of entries in memory.
type memFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
mu sync.Mutex
entries map[string]*filer_pb.Entry // keyed by directory + "/" + name
// afterList runs once a listing is sent, as another process would act.
afterList func()
}
func newMemFiler() *memFiler { return &memFiler{entries: map[string]*filer_pb.Entry{}} }
func (f *memFiler) put(dir string, entry *filer_pb.Entry) {
f.mu.Lock()
defer f.mu.Unlock()
f.entries[dir+"/"+entry.Name] = entry
}
func (f *memFiler) names(dir string) []string {
f.mu.Lock()
defer f.mu.Unlock()
var names []string
for key, e := range f.entries {
if key == dir+"/"+e.Name {
names = append(names, e.Name)
}
}
sort.Strings(names)
return names
}
func (f *memFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
if e, ok := f.entries[req.Directory+"/"+req.Name]; ok {
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
}
return nil, status.Error(codes.NotFound, "not found")
}
func (f *memFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
key := req.Directory + "/" + req.Entry.Name
for _, c := range req.GetCondition().GetClauses() {
if _, exists := f.entries[key]; exists && c.Kind == filer_pb.WriteCondition_IF_NOT_EXISTS {
return &filer_pb.CreateEntryResponse{Error: "exists", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
}
}
f.entries[key] = req.Entry
return &filer_pb.CreateEntryResponse{}, nil
}
func (f *memFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.entries, req.Directory+"/"+req.Name)
return &filer_pb.DeleteEntryResponse{}, nil
}
func (f *memFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
for _, name := range f.names(req.Directory) {
if name <= req.StartFromFileName {
continue
}
f.mu.Lock()
e := f.entries[req.Directory+"/"+name]
f.mu.Unlock()
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: e}); err != nil {
return err
}
}
if f.afterList != nil {
f.afterList()
}
return nil
}
func (f *memFiler) satisfied(cond *filer_pb.WriteCondition, key string) bool {
current, exists := f.entries[key]
for _, c := range cond.GetClauses() {
switch c.Kind {
case filer_pb.WriteCondition_IF_NOT_EXISTS:
if exists {
return false
}
case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
if !exists || !proto.Equal(current, c.ExpectedEntry) {
return false
}
}
}
return true
}
func (f *memFiler) ObjectTransaction(_ context.Context, req *filer_pb.ObjectTransactionRequest) (*filer_pb.ObjectTransactionResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
conditionKey := req.LockKey
if req.ConditionKey != "" {
conditionKey = req.ConditionKey
}
if !f.satisfied(req.Condition, conditionKey) {
return &filer_pb.ObjectTransactionResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
}
for i, m := range req.Mutations {
key := m.Directory + "/" + m.Name
if m.Type == filer_pb.ObjectMutation_PUT {
key = m.Directory + "/" + m.Entry.Name
}
if !f.satisfied(m.Condition, key) {
failed := int32(i)
return &filer_pb.ObjectTransactionResponse{Error: "precondition failed", ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED, FailedMutation: &failed}, nil
}
if m.Type == filer_pb.ObjectMutation_PUT {
f.entries[key] = m.Entry
} else {
delete(f.entries, key)
}
}
return &filer_pb.ObjectTransactionResponse{}, nil
}
// sha1Names holds the SHA-1 names, which fips140=only cannot compute.
var sha1Names = map[string]string{
"session-1": "96c71081c21826a93aad95a48dad46723ef67a06.json",
"arn:aws:iam::123456789012:oidc-provider/issuer.example.com": "a34a5d627a0d07f6793272fe8fae86d3957de95f.json",
}
// otherModeName is the name the other fips140 mode derives for key.
func otherModeName(key string) string {
if fips140.Enforced() {
return sha1Names[key]
}
sum := sha256.Sum256([]byte(key))
return hex.EncodeToString(sum[:20]) + ".json"
}
func storeAs(t *testing.T, filer *memFiler, dir, name string, record any) {
data, err := json.Marshal(record)
require.NoError(t, err)
filer.put(dir, &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{FileMode: 0o600}, Content: data})
}
func TestRevocationWrittenInTheOtherModeStillRevokes(t *testing.T) {
ctx := context.Background()
filer := newMemFiler()
store := NewFilerSessionRevocationStore(nil, startTestFiler(t, filer))
store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
storeAs(t, filer, store.basePath, otherModeName("session-1"), &RevocationEntry{JTI: "session-1", ExpiresAt: time.Now().Add(time.Hour)})
revoked, err := store.IsRevoked(ctx, "", "session-1")
require.NoError(t, err)
require.True(t, revoked)
require.Equal(t, []string{store.fileName("session-1")}, filer.names(store.basePath))
}
func TestProviderStoredInTheOtherModeCanBeDeleted(t *testing.T) {
ctx := context.Background()
filer := newMemFiler()
store := NewFilerOIDCProviderStore(nil, startTestFiler(t, filer))
store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
const arn = "arn:aws:iam::123456789012:oidc-provider/issuer.example.com"
const issuer = "https://issuer.example.com"
storeAs(t, filer, store.basePath, otherModeName(arn), newRecord(arn, issuer))
got, err := store.GetProviderByARN(ctx, "", arn)
require.NoError(t, err)
require.Equal(t, issuer, got.URL)
require.NoError(t, store.DeleteProvider(ctx, "", arn))
_, err = store.GetProviderByIssuer(ctx, "", issuer)
require.Error(t, err, "a deleted provider must not stay trusted")
require.Empty(t, filer.names(store.basePath))
}
// A record already under this mode's name is newer than its other-mode copy.
func TestRenameKeepsTheRecordUnderThisModesName(t *testing.T) {
ctx := context.Background()
filer := newMemFiler()
store := NewFilerOIDCProviderStore(nil, startTestFiler(t, filer))
store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
const arn = "arn:aws:iam::123456789012:oidc-provider/issuer.example.com"
storeAs(t, filer, store.basePath, otherModeName(arn), newRecord(arn, "https://old.example.com"))
storeAs(t, filer, store.basePath, store.fileName(arn), newRecord(arn, "https://new.example.com"))
records, err := store.ListProviders(ctx, "")
require.NoError(t, err)
require.Len(t, records, 1)
require.Equal(t, "https://new.example.com", records[0].URL)
}
// Another process renamed the record and deleted the provider after this one
// listed the directory: the stale listing must not bring the provider back.
func TestRenameDoesNotRestoreARecordMovedAndDeletedMeanwhile(t *testing.T) {
ctx := context.Background()
filer := newMemFiler()
store := NewFilerOIDCProviderStore(nil, startTestFiler(t, filer))
store.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
const arn = "arn:aws:iam::123456789012:oidc-provider/issuer.example.com"
storeAs(t, filer, store.basePath, otherModeName(arn), newRecord(arn, "https://issuer.example.com"))
filer.afterList = func() {
filer.afterList = nil
filer.mu.Lock()
delete(filer.entries, store.basePath+"/"+otherModeName(arn))
filer.mu.Unlock()
}
_, err := store.GetProviderByARN(ctx, "", arn)
require.ErrorIs(t, err, ErrOIDCProviderNotFound)
require.Empty(t, filer.names(store.basePath))
}
-4
View File
@@ -4,7 +4,6 @@ import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/fips140"
"crypto/rsa"
"crypto/sha1"
"crypto/tls"
@@ -259,9 +258,6 @@ func normalizeThumbprints(in []string) map[string]struct{} {
// self-signed deployments and skip-verify test setups. When the chain has not
// been built (InsecureSkipVerify), we fall back to PeerCertificates.
func verifyThumbprintMatch(cs tls.ConnectionState, expected map[string]struct{}) error {
if fips140.Enforced() {
return fmt.Errorf("OIDC TLS thumbprints are SHA-1, which GODEBUG=fips140=only refuses")
}
candidates := collectThumbprintCandidates(cs)
for _, c := range candidates {
sum := sha1.Sum(c.Raw)
+2 -2
View File
@@ -301,8 +301,8 @@ func fetchChunkFromPeer(ctx context.Context, dial MountPeerDialer, peerAddr, fid
// Compare raw digests so we don't reject every valid peer response
// because of an encoding mismatch.
if expectedETag != "" {
got := util.Md5(buf)
if !etagMatchesMD5(expectedETag, got) {
got := md5.Sum(buf)
if !etagMatchesMD5(expectedETag, got[:]) {
return nil, fmt.Errorf("etag mismatch: peer=%s got=%x want=%s", peerAddr, got, expectedETag)
}
}
+3 -3
View File
@@ -3,6 +3,7 @@ package operation
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"errors"
"fmt"
@@ -16,7 +17,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
@@ -64,7 +64,7 @@ func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUpl
func uploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption, prepare func([]byte, *UploadOption) (*preparedUpload, error)) (*ChunkedUploadResult, error) {
md5Hash := util.NewMd5()
md5Hash := md5.New()
var partReader = io.TeeReader(reader, md5Hash)
var fileChunks []*filer_pb.FileChunk
@@ -183,7 +183,7 @@ uploadLoop:
}()
// Hash and prepare only this bounded chunk before minting its volume JWT.
chunkMd5 := util.Md5(buf.Bytes())
chunkMd5 := md5.Sum(buf.Bytes())
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
var prepared *preparedUpload
if opt.UploadFunc == nil {
+6 -15
View File
@@ -3,7 +3,7 @@ package operation
import (
"bytes"
"context"
"crypto/fips140"
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
@@ -316,7 +316,8 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
// verifies). Under cipher the server never sees the header, but the digest
// still becomes the chunk ETag.
if uploadOption.WantMd5 && uploadOption.Md5 == "" {
uploadOption.Md5 = base64.StdEncoding.EncodeToString(util.Md5(data))
digest := md5.Sum(data)
uploadOption.Md5 = base64.StdEncoding.EncodeToString(digest[:])
}
fileId, uploadResult, err = uploader.uploadWithRetryData(func() (fileId string, host string, auth security.EncodedJwt, fsync bool, err error) {
@@ -615,7 +616,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
if option.IsInputCompressed {
h.Set("Content-Encoding", "gzip")
}
if option.Md5 != "" && !fips140.Enforced() {
if option.Md5 != "" {
h.Set("Content-MD5", option.Md5)
}
@@ -755,7 +756,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps})
}
ret.ETag = etag
ret.ContentMd5 = uploadedContentMd5(resp, option)
ret.ContentMd5 = resp.Header.Get("Content-MD5")
return &ret, nil
}
@@ -792,7 +793,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps})
}
ret.ETag = etag
ret.ContentMd5 = uploadedContentMd5(resp, option)
ret.ContentMd5 = resp.Header.Get("Content-MD5")
return &ret, nil
}
@@ -868,13 +869,3 @@ func (b *watchdogBody) Close() error {
b.sentAt.Store(time.Now().UnixNano())
return b.ReadCloser.Close()
}
// uploadedContentMd5 is the Content-MD5 the volume server checked and echoed.
// Under fips140=only the digest is not an MD5 and is never sent, so the
// caller's own digest stands.
func uploadedContentMd5(resp *http.Response, option *UploadOption) string {
if fips140.Enforced() {
return option.Md5
}
return resp.Header.Get("Content-MD5")
}
+1 -2
View File
@@ -1,7 +1,6 @@
package S3Sink
import (
"crypto/fips140"
"encoding/base64"
"fmt"
"mime"
@@ -222,7 +221,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
if entry.Attributes.Mime != "" {
uploadInput.ContentType = aws.String(entry.Attributes.Mime)
}
if len(entry.Attributes.Md5) > 0 && !fips140.Enforced() {
if len(entry.Attributes.Md5) > 0 {
uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
}
_, err = uploader.Upload(&uploadInput)
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"cmp"
"container/heap"
"context"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"encoding/hex"
@@ -1964,7 +1965,7 @@ func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPart
glog.Warningf("calculateMultipartETag: failed to decode etag '%s' for part %d: %v", etag, partNumber, err)
}
}
return fmt.Sprintf("%x-%d", util.Md5(etags), len(completedPartNumbers))
return fmt.Sprintf("%x-%d", md5.Sum(etags), len(completedPartNumbers))
}
func decodePartChecksum(partNumber int, entries []*filer_pb.Entry, checksumHeaderName string) ([]byte, *filer_pb.Entry, error) {
+1 -10
View File
@@ -1,9 +1,6 @@
package s3api
import (
"bytes"
"crypto/fips140"
)
import "crypto/fips140"
// fips140Enforced reports whether the process runs with GODEBUG=fips140=only.
// In that mode MD5, SHA-1 and HMAC-SHA1 panic, so SigV2, SSE-C and the SHA-1
@@ -11,9 +8,3 @@ import (
// It is a variable so tests can enable the refusals without running the whole
// test binary under fips140=only.
var fips140Enforced = fips140.Enforced
// contentMd5Matches compares a body digest with the client's Content-MD5.
// Under fips140=only the digest is not an MD5, so Content-MD5 goes unchecked.
func contentMd5Matches(digest, contentMd5 []byte) bool {
return fips140Enforced() || bytes.Equal(digest, contentMd5)
}
+3 -7
View File
@@ -131,9 +131,7 @@ func bindSignedPayload(r *http.Request, hashedPayload string) s3err.ErrorCode {
if err != nil || len(decoded) != md5.Size {
return s3err.ErrInvalidDigest
}
if !fips140Enforced() {
contentMd5 = decoded
}
contentMd5 = decoded
}
if expected == nil && contentMd5 == nil {
return s3err.ErrNone
@@ -142,10 +140,8 @@ func bindSignedPayload(r *http.Request, hashedPayload string) s3err.ErrorCode {
if emptySum := sha256.Sum256(nil); expected != nil && !bytes.Equal(expected, emptySum[:]) {
return s3err.ErrContentSHA256Mismatch
}
if contentMd5 != nil {
if emptySum := md5.Sum(nil); !bytes.Equal(contentMd5, emptySum[:]) {
return s3err.ErrBadDigest
}
if emptySum := md5.Sum(nil); contentMd5 != nil && !bytes.Equal(contentMd5, emptySum[:]) {
return s3err.ErrBadDigest
}
return s3err.ErrNone
}
-8
View File
@@ -263,11 +263,3 @@ func TestPrebufferedSignedBodyChecksContentMd5(t *testing.T) {
_, err = io.ReadAll(req.Body)
assert.ErrorIs(t, err, errContentMd5Mismatch)
}
func TestBodylessSignedRequestSkipsContentMd5UnderFIPS140Only(t *testing.T) {
setFIPS140Enforced(t, true)
emptySha256 := sha256.Sum256(nil)
req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:9000/bucket/key", nil)
req.Header.Set("Content-Md5", "AAAAAAAAAAAAAAAAAAAAAA==")
assert.Equal(t, s3err.ErrNone, bindSignedPayload(req, hex.EncodeToString(emptySha256[:])))
}
+1 -6
View File
@@ -511,12 +511,7 @@ func (s3a *S3ApiServer) checkDirectoryObject(bucket, object string) (*filer_pb.E
func (s3a *S3ApiServer) resolveObjectEntry(bucket, object, versionId string) (*filer_pb.Entry, error) {
// Check if versioning is configured
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
// Keep filer_pb.ErrNotFound in the error chain so callers that only
// check not-found do not misclassify a missing bucket as internal error.
return nil, fmt.Errorf("%w: %w", ErrBucketNotFound, err)
}
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
glog.Errorf("resolveObjectEntry: error checking versioning config for %s: %v", bucket, err)
return nil, err
}
+4 -4
View File
@@ -1263,7 +1263,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
// For zero-size files or invalid ranges, create an empty part with size 0
dstEntry.Attributes.FileSize = 0
dstEntry.Chunks = nil
dstEntry.Attributes.Md5 = util.Md5(nil)
dstEntry.Attributes.Md5 = md5.New().Sum(nil)
} else {
// Copy chunks that overlap with the range
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, startOffset, endOffset, dstAssignPath)
@@ -1630,7 +1630,7 @@ func (s3a *S3ApiServer) copySingleChunkForRange(originalChunk, rangeChunk *filer
// its ETag has to be recomputed; a whole-chunk copy reuses it.
var digest hash.Hash
if !isFullChunk {
digest = util.NewMd5()
digest = md5.New()
}
if err := s3a.streamCopyChunkRange(context.Background(), srcUrl, fileId, offsetInChunk, int64(rangeChunk.Size), isFullChunk, assignResult, digest); err != nil {
return nil, fmt.Errorf("stream chunk range: %w", err)
@@ -1856,7 +1856,7 @@ func (s3a *S3ApiServer) copyChunksForRange(entry *filer_pb.Entry, startOffset, e
// md5Zeros hashes size zero bytes without a volume read — a copied range
// that overlaps no source chunk assembles to all zeros.
func md5Zeros(size int64) []byte {
h := util.NewMd5()
h := md5.New()
var zeros [1 << 16]byte
for size > 0 {
n := min(size, int64(len(zeros)))
@@ -1878,7 +1878,7 @@ func (s3a *S3ApiServer) copiedRangeMd5(ctx context.Context, entry *filer_pb.Entr
return nil, err
}
defer reader.Close()
h := util.NewMd5()
h := md5.New()
if _, err := io.Copy(h, reader); err != nil {
return nil, err
}
@@ -15,7 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
@@ -447,7 +446,7 @@ func TestWriteEmptyCopyPartStoresMd5(t *testing.T) {
if code != s3err.ErrNone {
t.Fatalf("writeEmptyCopyPart: %v", code)
}
emptyMD5Hex := fmt.Sprintf("%x", util.Md5(nil))
const emptyMD5Hex = "d41d8cd98f00b204e9800998ecf8427e"
if etag != emptyMD5Hex {
t.Fatalf("writeEmptyCopyPart etag = %q, want %q", etag, emptyMD5Hex)
}
@@ -2,6 +2,7 @@ package s3api
import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
@@ -19,7 +20,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// errCopySourceSSEUnsupported is returned by openSourcePlaintextReader when
@@ -421,20 +421,19 @@ func (s3a *S3ApiServer) writeEmptyCopyPart(dstBucket, dstObject, uploadID string
return "", s3err.ErrInternalError
}
}
emptyMd5 := util.Md5(nil)
emptyETag := fmt.Sprintf("%x", emptyMd5)
const emptyMD5Hex = "d41d8cd98f00b204e9800998ecf8427e"
if err := upload.mkFile(uploadDir, partName, nil, func(e *filer_pb.Entry) {
if e.Attributes == nil {
e.Attributes = &filer_pb.FuseAttributes{}
}
e.Attributes.FileSize = 0
e.Attributes.Md5 = emptyMd5
e.Attributes.Md5 = md5.New().Sum(nil)
if e.Extended == nil {
e.Extended = map[string][]byte{}
}
e.Extended[s3_constants.ExtETagKey] = []byte(emptyETag)
e.Extended[s3_constants.ExtETagKey] = []byte(emptyMD5Hex)
}); err != nil {
return "", s3err.ErrInternalError
}
return emptyETag, s3err.ErrNone
return emptyMD5Hex, s3err.ErrNone
}
+1 -1
View File
@@ -102,7 +102,7 @@ func (s3a *S3ApiServer) extractZipObject(w http.ResponseWriter, r *http.Request,
requestQuery := parseRequestQuery(r)
archiveMd5 := r.Header.Get("Content-Md5")
var archiveMd5Hash hash.Hash
if archiveMd5 != "" && !fips140Enforced() {
if archiveMd5 != "" {
archiveMd5Hash = md5.New()
dataReader = io.TeeReader(dataReader, archiveMd5Hash)
}
@@ -1,358 +0,0 @@
package s3api
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"path"
"strings"
"sync"
"testing"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"google.golang.org/protobuf/proto"
)
func newMissingBucketConditionalTestServer(t *testing.T, filer filer_pb.SeaweedFilerServer) *S3ApiServer {
t.Helper()
s3a := newHeadBucketTestServer(t, filer)
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
return s3a
}
func TestResolveObjectEntryMissingBucketError(t *testing.T) {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
entry, err := s3a.resolveObjectEntry("missing-bucket", "key", "")
if entry != nil {
t.Fatalf("entry = %v, want nil", entry)
}
if !errors.Is(err, ErrBucketNotFound) {
t.Fatalf("err = %v, want errors.Is(err, ErrBucketNotFound)", err)
}
if !errors.Is(err, filer_pb.ErrNotFound) {
t.Fatalf("err = %v, want errors.Is(err, filer_pb.ErrNotFound)", err)
}
}
func TestPutObjectConditionalHeadersOnMissingBucket(t *testing.T) {
const bucket = "missing-bucket"
const object = "obj"
cases := []struct {
name string
autoCreateBucket bool
header string
value string
}{
{
name: "If-Match ETag with AutoCreateBucket=false",
autoCreateBucket: false,
header: "If-Match",
value: `"00000000000000000000000000000bad"`,
},
{
name: "If-Match wildcard with AutoCreateBucket=false",
autoCreateBucket: false,
header: "If-Match",
value: "*",
},
{
name: "If-Match ETag with AutoCreateBucket=true",
autoCreateBucket: true,
header: "If-Match",
value: `"00000000000000000000000000000bad"`,
},
{
name: "If-None-Match wildcard with AutoCreateBucket=false",
autoCreateBucket: false,
header: "If-None-Match",
value: "*",
},
{
name: "If-Unmodified-Since with AutoCreateBucket=false",
autoCreateBucket: false,
header: "If-Unmodified-Since",
value: time.Now().UTC().Add(24 * time.Hour).Format(http.TimeFormat),
},
{
name: "If-Modified-Since with AutoCreateBucket=false",
autoCreateBucket: false,
header: "If-Modified-Since",
value: time.Now().UTC().Add(-24 * time.Hour).Format(http.TimeFormat),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
s3a.option.AutoCreateBucket = tc.autoCreateBucket
req := httptest.NewRequest(http.MethodPut, "/"+bucket+"/"+object, strings.NewReader("payload"))
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
req.Header.Set(tc.header, tc.value)
rr := httptest.NewRecorder()
s3a.PutObjectHandler(rr, req)
if rr.Code != http.StatusNotFound || !strings.Contains(rr.Body.String(), "NoSuchBucket") {
t.Fatalf("status = %d body = %q, want 404 NoSuchBucket", rr.Code, rr.Body.String())
}
})
}
// A non-If-Match condition passes against an absent object, so the request must
// reach the handler's auto-create gate, which validates the bucket name. Answering
// NoSuchBucket here would report a malformed name as merely missing.
t.Run("malformed bucket name with If-None-Match wildcard reports InvalidBucketName", func(t *testing.T) {
const malformedBucket = "ab"
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
s3a.option.AutoCreateBucket = false
req := httptest.NewRequest(http.MethodPut, "/"+malformedBucket+"/"+object, strings.NewReader("payload"))
req = mux.SetURLVars(req, map[string]string{"bucket": malformedBucket, "object": object})
req.Header.Set("If-None-Match", "*")
rr := httptest.NewRecorder()
s3a.PutObjectHandler(rr, req)
if rr.Code != http.StatusBadRequest || !strings.Contains(rr.Body.String(), "<Code>InvalidBucketName</Code>") {
t.Fatalf("status = %d body = %q, want 400 InvalidBucketName", rr.Code, rr.Body.String())
}
})
t.Run("checkConditionalHeaders with AutoCreateBucket=true leaves auto-create to the handler", func(t *testing.T) {
future := time.Now().UTC().Add(24 * time.Hour).Format(http.TimeFormat)
past := time.Now().UTC().Add(-24 * time.Hour).Format(http.TimeFormat)
for _, header := range []struct {
name string
value string
}{
{"If-None-Match", "*"},
{"If-Unmodified-Since", future},
{"If-Modified-Since", past},
} {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
s3a.option.AutoCreateBucket = true
req := httptest.NewRequest(http.MethodPut, "/"+bucket+"/"+object, strings.NewReader("payload"))
req.Header.Set(header.name, header.value)
if errCode := s3a.checkConditionalHeaders(req, bucket, object); errCode != s3err.ErrNone {
t.Fatalf("%s: errCode = %v, want ErrNone", header.name, errCode)
}
}
})
}
func TestReadConditionalHeadersOnMissingBucket(t *testing.T) {
const bucket = "missing-bucket"
const object = "obj"
future := time.Now().UTC().Add(24 * time.Hour).Format(http.TimeFormat)
past := time.Now().UTC().Add(-24 * time.Hour).Format(http.TimeFormat)
cases := []struct {
name string
header string
value string
}{
{name: "If-Match ETag", header: "If-Match", value: `"00000000000000000000000000000bad"`},
{name: "If-Match wildcard", header: "If-Match", value: "*"},
{name: "If-Unmodified-Since", header: "If-Unmodified-Since", value: future},
{name: "If-None-Match wildcard", header: "If-None-Match", value: "*"},
{name: "If-Modified-Since", header: "If-Modified-Since", value: past},
}
for _, tc := range cases {
t.Run("GET/"+tc.name, func(t *testing.T) {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/"+object, nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
req.Header.Set(tc.header, tc.value)
rr := httptest.NewRecorder()
s3a.GetObjectHandler(rr, req)
if rr.Code != http.StatusNotFound || !strings.Contains(rr.Body.String(), "NoSuchBucket") {
t.Fatalf("GET status = %d body = %q, want 404 NoSuchBucket", rr.Code, rr.Body.String())
}
})
t.Run("HEAD/"+tc.name, func(t *testing.T) {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
req := httptest.NewRequest(http.MethodHead, "/"+bucket+"/"+object, nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
req.Header.Set(tc.header, tc.value)
res := s3a.checkConditionalHeadersForReads(req, bucket, object)
if res.ErrorCode != s3err.ErrNoSuchBucket {
t.Fatalf("checkConditionalHeadersForReads ErrorCode = %v, want ErrNoSuchBucket", res.ErrorCode)
}
rr := httptest.NewRecorder()
s3a.HeadObjectHandler(rr, req)
if rr.Code != http.StatusNotFound {
t.Fatalf("HEAD status = %d, want 404", rr.Code)
}
if rr.Body.Len() != 0 {
t.Fatalf("HEAD response must have an empty body, got %q", rr.Body.String())
}
})
}
t.Run("GET versionId with If-Match", func(t *testing.T) {
const versionedObject = "some/obj"
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
checkReq := httptest.NewRequest(http.MethodGet, "/"+bucket+"/"+versionedObject+"?versionId=v1", nil)
checkReq = mux.SetURLVars(checkReq, map[string]string{"bucket": bucket, "object": versionedObject})
checkReq.Header.Set("If-Match", `"00000000000000000000000000000bad"`)
if res := s3a.checkConditionalHeadersForReads(checkReq, bucket, versionedObject); res.ErrorCode != s3err.ErrNoSuchBucket {
t.Fatalf("checkConditionalHeadersForReads ErrorCode = %v, want ErrNoSuchBucket", res.ErrorCode)
}
req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/"+versionedObject+"?versionId=v1", nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": versionedObject})
req.Header.Set("If-Match", `"00000000000000000000000000000bad"`)
rr := httptest.NewRecorder()
s3a.GetObjectHandler(rr, req)
if rr.Code != http.StatusNotFound || !strings.Contains(rr.Body.String(), "NoSuchBucket") {
t.Fatalf("GET status = %d body = %q, want 404 NoSuchBucket", rr.Code, rr.Body.String())
}
})
}
func TestAnonymousGetConditionalOnMissingBucketReturnsAccessDenied(t *testing.T) {
const bucket = "missing-bucket"
const object = "obj"
s3a, _ := newAnonymousReadTestServer(t)
req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/"+object, nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
req.Header.Set("If-Match", `"00000000000000000000000000000bad"`)
rr := httptest.NewRecorder()
s3a.AuthWithPublicRead(s3a.GetObjectHandler, s3_constants.ACTION_READ)(rr, req)
if rr.Code != http.StatusForbidden || !strings.Contains(rr.Body.String(), "AccessDenied") {
t.Fatalf("anonymous GET status = %d body = %q, want 403 AccessDenied", rr.Code, rr.Body.String())
}
}
// A caller that may list the bucket can already learn which keys exist, so the
// missing bucket is answered as such rather than masked as AccessDenied.
func TestAnonymousGetConditionalOnMissingBucketListAllowed(t *testing.T) {
const bucket = "missing-bucket"
const object = "obj"
future := time.Now().UTC().Add(24 * time.Hour).Format(http.TimeFormat)
for _, header := range []struct {
name string
value string
}{
{"If-Match", `"00000000000000000000000000000bad"`},
{"If-Unmodified-Since", future},
} {
s3a := newMissingBucketConditionalTestServer(t, &fakeLookupFiler{})
s3a.iam = &IdentityAccessManagement{isAuthEnabled: true}
s3a.iam.identityAnonymous = &Identity{Name: s3_constants.AccountAnonymousId, Actions: []Action{s3_constants.ACTION_LIST}}
s3a.policyEngine = NewBucketPolicyEngine()
s3a.iam.policyEngine = s3a.policyEngine
req := httptest.NewRequest(http.MethodGet, "/"+bucket+"/"+object, nil)
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object})
req.Header.Set(header.name, header.value)
rr := httptest.NewRecorder()
s3a.AuthWithPublicRead(s3a.GetObjectHandler, s3_constants.ACTION_READ)(rr, req)
if rr.Code != http.StatusNotFound || !strings.Contains(rr.Body.String(), "NoSuchBucket") {
t.Fatalf("%s: anonymous GET status = %d body = %q, want 404 NoSuchBucket", header.name, rr.Code, rr.Body.String())
}
headReq := httptest.NewRequest(http.MethodHead, "/"+bucket+"/"+object, nil)
headReq = mux.SetURLVars(headReq, map[string]string{"bucket": bucket, "object": object})
headReq.Header.Set(header.name, header.value)
headRR := httptest.NewRecorder()
s3a.AuthWithPublicRead(s3a.HeadObjectHandler, s3_constants.ACTION_READ)(headRR, headReq)
if headRR.Code != http.StatusNotFound {
t.Fatalf("%s: anonymous HEAD status = %d, want 404", header.name, headRR.Code)
}
if headRR.Body.Len() != 0 {
t.Fatalf("%s: anonymous HEAD response must have an empty body, got %q", header.name, headRR.Body.String())
}
}
}
// autoCreateMissingBucketFiler keeps created entries, so a handler can resolve
// the bucket it just auto-created.
type autoCreateMissingBucketFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
mu sync.Mutex
entries map[string]*filer_pb.Entry
}
func (f *autoCreateMissingBucketFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
entry := f.entries[path.Join(req.Directory, req.Name)]
if entry == nil {
return &filer_pb.LookupDirectoryEntryResponse{}, nil
}
return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
}
func (f *autoCreateMissingBucketFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
entry := proto.Clone(req.Entry).(*filer_pb.Entry)
f.entries[path.Join(req.Directory, entry.Name)] = entry
return &filer_pb.CreateEntryResponse{}, nil
}
// A conditional PUT the gate would auto-create for is not answered NoSuchBucket:
// the directory marker and the bucket are created by the same request.
func TestPutDirectoryMarkerConditionalAutoCreateMissingBucket(t *testing.T) {
const bucket = "missing-bucket"
filer := &autoCreateMissingBucketFiler{entries: make(map[string]*filer_pb.Entry)}
s3a := newMissingBucketConditionalTestServer(t, filer)
s3a.option.AutoCreateBucket = true
s3a.iam = &IdentityAccessManagement{}
req := httptest.NewRequest(http.MethodPut, "/"+bucket+"/dir/", strings.NewReader(""))
req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": "dir/"})
req = req.WithContext(recordIdentityInContext(req, &Identity{Name: "admin", Actions: []Action{s3_constants.ACTION_ADMIN}}))
req.Header.Set("If-None-Match", "*")
rr := httptest.NewRecorder()
s3a.PutObjectHandler(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("PUT status = %d body = %q, want 200", rr.Code, rr.Body.String())
}
filer.mu.Lock()
bucketEntry := filer.entries["/buckets/"+bucket]
markerEntry := filer.entries["/buckets/"+bucket+"/dir"]
filer.mu.Unlock()
if bucketEntry == nil || !bucketEntry.IsDirectory {
t.Fatalf("bucket entry = %v, want a created directory", bucketEntry)
}
if markerEntry == nil || !markerEntry.IsDirectory {
t.Fatalf("object entry = %v, want a created directory marker", markerEntry)
}
}
@@ -1,6 +1,7 @@
package s3api
import (
"crypto/sha1"
"encoding/base64"
"encoding/json"
"encoding/xml"
@@ -22,7 +23,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
@@ -605,7 +605,7 @@ func (s3a *S3ApiServer) genPartUploadPath(bucket, uploadID string, partID int) s
func (s3a *S3ApiServer) generateUploadID(object string) string {
object = strings.TrimPrefix(object, "/")
h := util.NewSha1()
h := sha1.New()
h.Write([]byte(object))
return fmt.Sprintf("%x", h.Sum(nil))
}
+9 -16
View File
@@ -3,6 +3,7 @@ package s3api
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
@@ -30,7 +31,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/constants"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"google.golang.org/grpc/codes"
@@ -234,9 +234,9 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
}
}
// ETag of the marker content, empty or not
dirMd5 := util.Md5(dirContent)
if expectedMd5, _ := validateContentMd5(r.Header); len(expectedMd5) > 0 && !contentMd5Matches(dirMd5, expectedMd5) {
// Compute MD5 for ETag (md5.Sum of nil/empty = MD5 of empty content)
dirMd5 := md5.Sum(dirContent)
if expectedMd5, _ := validateContentMd5(r.Header); len(expectedMd5) > 0 && !bytes.Equal(dirMd5[:], expectedMd5) {
s3err.WriteErrorResponse(w, r, s3err.ErrBadDigest)
return
}
@@ -287,7 +287,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
entry.Content = dirContent
}
entry.Attributes.Mime = objectContentType
entry.Attributes.Md5 = dirMd5
entry.Attributes.Md5 = dirMd5[:]
// Store ETag in extended attributes for consistency with regular objects
if entry.Extended == nil {
@@ -611,7 +611,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
// Each part is encrypted independently, then decrypted using metadata during GET
partOffset := int64(0)
plaintextHash := util.NewMd5()
plaintextHash := md5.New()
dataReader = io.TeeReader(dataReader, plaintextHash)
// Parse the request's query string once. AWS SDK presigners hoist signed
@@ -800,7 +800,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
s3a.deleteOrphanedChunks(chunkResult.FileChunks)
return "", s3err.ErrInvalidDigest, SSEResponseMetadata{}
}
if !contentMd5Matches(md5Sum, expectedMd5) {
if !bytes.Equal(md5Sum, expectedMd5) {
glog.Warningf("putToFiler: Checksum verification failed, attempting to cleanup %d orphaned chunks", len(chunkResult.FileChunks))
s3a.deleteOrphanedChunks(chunkResult.FileChunks)
return "", s3err.ErrBadDigest, SSEResponseMetadata{}
@@ -2986,12 +2986,7 @@ func (s3a *S3ApiServer) checkConditionalHeaders(r *http.Request, bucket, object
// This ensures we check conditions against the LATEST version, not a null version.
entry, err := s3a.resolveObjectEntry(bucket, object, "")
if err != nil {
if errors.Is(err, ErrBucketNotFound) {
if headers.ifMatch != "" {
return s3err.ErrNoSuchBucket
}
entry = nil
} else if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
entry = nil
} else {
glog.Errorf("checkConditionalHeaders: error resolving object entry for %s/%s: %v", bucket, object, err)
@@ -3109,9 +3104,7 @@ func (s3a *S3ApiServer) checkConditionalHeadersForReads(r *http.Request, bucket,
versionId := r.URL.Query().Get("versionId")
entry, err := s3a.resolveObjectEntry(bucket, object, versionId)
if err != nil {
if errors.Is(err, ErrBucketNotFound) {
return ConditionalHeaderResult{ErrorCode: s3err.ErrNoSuchBucket, Entry: nil}
} else if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
entry = nil
} else {
glog.Errorf("checkConditionalHeadersForReads: error resolving object entry for %s/%s: %v", bucket, object, err)
+5 -3
View File
@@ -7,6 +7,7 @@ package s3api
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"encoding/xml"
"errors"
@@ -20,7 +21,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/version"
)
@@ -259,7 +259,8 @@ func (s3a *S3ApiServer) handleSOSAPIGetObject(w http.ResponseWriter, r *http.Req
}
// Calculate ETag from content
etag := hex.EncodeToString(util.Md5(xmlData))
hash := md5.Sum(xmlData)
etag := hex.EncodeToString(hash[:])
// Set ETag header manually as ServeContent doesn't calculate it automatically
w.Header().Set("ETag", "\""+etag+"\"")
@@ -290,7 +291,8 @@ func (s3a *S3ApiServer) handleSOSAPIHeadObject(w http.ResponseWriter, r *http.Re
}
// Calculate ETag from content
etag := hex.EncodeToString(util.Md5(xmlData))
hash := md5.Sum(xmlData)
etag := hex.EncodeToString(hash[:])
// Set response headers (no body for HEAD)
w.Header().Set("Content-Type", "application/xml")
+7 -9
View File
@@ -231,7 +231,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
return ownerResp, nil
}
} else if req.IsMoved && routed {
if err := fs.checkForwardedWrite(ctx, entryRouteKey(util.NewFullPath(req.Directory, req.Entry.Name))); err != nil {
if err := fs.checkMovedMarker(ctx, req.IsMoved, fs.writeOwner(entryRouteKey(util.NewFullPath(req.Directory, req.Entry.Name)))); err != nil {
return &filer_pb.CreateEntryResponse{}, err
}
}
@@ -347,12 +347,10 @@ func (fs *FilerServer) ObjectTransaction(ctx context.Context, req *filer_pb.Obje
// Route-by-key: if this filer is not the ring owner of route_key, forward the
// whole transaction to the owner so its per-path lock is the single
// serialization point — even when the caller's ring view was stale. is_moved
// bounds this to one hop: a forwarded transaction is applied only by the
// owner in this filer's ring, and refused with a status the sender retries.
if req.RouteKey != "" && req.IsMoved {
if err := fs.checkForwardedWrite(ctx, req.RouteKey); isNotWriteOwner(err) {
return &filer_pb.ObjectTransactionResponse{}, err
} else if err != nil {
// bounds this to one hop: a forwarded transaction is applied locally, so two
// filers that disagree on the owner during a ring change cannot loop.
if req.RouteKey != "" {
if err := fs.checkMovedMarker(ctx, req.IsMoved, fs.writeOwner(req.RouteKey)); err != nil {
return &filer_pb.ObjectTransactionResponse{Error: err.Error()}, nil
}
}
@@ -956,7 +954,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
return ownerResp, nil
}
} else if req.IsMoved && routed {
if err := fs.checkForwardedWrite(ctx, entryRouteKey(util.FullPath(fullpath))); err != nil {
if err := fs.checkMovedMarker(ctx, req.IsMoved, fs.writeOwner(entryRouteKey(util.FullPath(fullpath)))); err != nil {
return &filer_pb.UpdateEntryResponse{}, err
}
}
@@ -1207,7 +1205,7 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
return ownerResp, nil
}
} else {
if err := fs.checkForwardedWrite(ctx, entryRouteKey(fullpath)); err != nil {
if err := fs.checkMovedMarker(ctx, req.IsMoved, fs.writeOwner(entryRouteKey(fullpath))); err != nil {
return &filer_pb.DeleteEntryResponse{}, err
}
}
@@ -78,20 +78,6 @@ func peerOwnedName(t *testing.T, fs *FilerServer) string {
return ""
}
// selfOwnedName returns a child name under /test whose entry the ring assigns
// to this filer.
func selfOwnedName(t *testing.T, fs *FilerServer) string {
t.Helper()
for i := 0; i < 4000; i++ {
name := fmt.Sprintf("obj-%d", i)
if fs.writeOwner(entryRouteKey(util.NewFullPath("/test", name))) == "" {
return name
}
}
t.Skip("no name owned by this filer")
return ""
}
func createReq(name string, oExcl bool) *filer_pb.CreateEntryRequest {
return &filer_pb.CreateEntryRequest{
Directory: "/test",
@@ -142,10 +128,11 @@ func ringPeerCtx() context.Context {
})
}
// A forwarded create is applied by the owner without forwarding it again.
// is_moved bounds forwarding to one hop: the owner applies it even if its own
// ring view says someone else owns the key.
func TestCreateEntryMovedAppliesLocally(t *testing.T) {
fs, store := createRouteServer(t)
name := selfOwnedName(t, fs)
name := peerOwnedName(t, fs)
req := createReq(name, true)
req.IsMoved = true
@@ -158,24 +145,6 @@ func TestCreateEntryMovedAppliesLocally(t *testing.T) {
}
}
// A filer whose own ring names a peer as the owner refuses a forwarded create
// rather than check it under its own lock: the sender's ring disagrees with
// this one, and the peer's lock may be arbitrating the same entry.
func TestCreateEntryMovedRefusedByNonOwner(t *testing.T) {
fs, store := createRouteServer(t)
name := peerOwnedName(t, fs)
req := createReq(name, true)
req.IsMoved = true
_, err := fs.CreateEntry(ringPeerCtx(), req)
if !isNotWriteOwner(err) {
t.Fatalf("a forwarded create on a non-owner must be refused as not the write owner, got err=%v", err)
}
if _, found := store.entries[string(util.NewFullPath("/test", name))]; found {
t.Fatal("a refused forward must not write locally")
}
}
// A caller can set is_moved, but without a ring member's connection it is not
// trusted: on a non-owner the request is refused rather than evaluated under
// the wrong lock or forwarded again, where it could cycle while rings disagree.
@@ -1,216 +0,0 @@
package weed_server
import (
"context"
"fmt"
"net"
"slices"
"sync"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// receivedUpdates counts the UpdateEntry requests a filer receives over gRPC,
// and how many of them another filer forwarded (is_moved).
type receivedUpdates struct {
mu sync.Mutex
updates, moved int
}
func (c *receivedUpdates) intercept(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
if r, ok := req.(*filer_pb.UpdateEntryRequest); ok {
c.mu.Lock()
c.updates++
if r.IsMoved {
c.moved++
}
c.mu.Unlock()
}
return handler(ctx, req)
}
func (c *receivedUpdates) counts() (updates, moved int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.updates, c.moved
}
// twoFilers is filers A and B on one shared store, each serving gRPC with its
// own ring view, so either can forward a routed write to the other.
type twoFilers struct {
store *renameTestStore
dial grpc.DialOption
addrA, addrB pb.ServerAddress
fsA, fsB *FilerServer
callsA, callsB *receivedUpdates
}
func newTwoFilers(t *testing.T) *twoFilers {
t.Helper()
c := &twoFilers{
store: newRenameTestStore(),
dial: grpc.WithTransportCredentials(insecure.NewCredentials()),
callsA: &receivedUpdates{},
callsB: &receivedUpdates{},
}
c.store.entries["/"] = newDirectoryEntry("/", 1)
serve := func(calls *receivedUpdates) (*FilerServer, pb.ServerAddress) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
host := pb.NewServerAddressWithGrpcPort(lis.Addr().String(), lis.Addr().(*net.TCPAddr).Port)
f := newRenameTestFiler(t, c.store)
f.Dlm = lock_manager.NewDistributedLockManager(host)
fs := &FilerServer{
filer: f,
option: &FilerOption{Host: host},
grpcDialOption: c.dial,
entryLockTable: util.NewLockTable[util.FullPath](),
}
srv := grpc.NewServer(grpc.UnaryInterceptor(calls.intercept))
filer_pb.RegisterSeaweedFilerServer(srv, fs)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(func() {
srv.Stop()
pb.InvalidateGrpcConnection(host.ToGrpcAddress())
})
return fs, host
}
c.fsA, c.addrA = serve(c.callsA)
c.fsB, c.addrB = serve(c.callsB)
return c
}
// setRing gives fs a ring built from the given snapshots, oldest first. Inside
// the cooling-off window of the last change a moved key keeps its prior owner.
func setRing(fs *FilerServer, window time.Duration, snapshots ...[]pb.ServerAddress) {
fs.filer.Dlm.LockRing = lock_manager.NewLockRing(window)
for i, servers := range snapshots {
fs.filer.Dlm.LockRing.SetSnapshot(slices.Clone(servers), int64(i+1))
}
}
// keyOwnedBy returns a test path whose write owner is ownerInA in A's ring and
// ownerInB in B's.
func (c *twoFilers) keyOwnedBy(t *testing.T, ownerInA, ownerInB pb.ServerAddress) util.FullPath {
t.Helper()
for i := 0; i < 4000; i++ {
p := util.FullPath(fmt.Sprintf("/forward-%d", i))
key := entryRouteKey(p)
if c.fsA.filer.Dlm.LockRing.WriteOwner(key) == ownerInA && c.fsB.filer.Dlm.LockRing.WriteOwner(key) == ownerInB {
return p
}
}
t.Fatalf("no path that A's ring gives to %s and B's ring gives to %s", ownerInA, ownerInB)
return ""
}
// seed stores p with the extended attribute v=seed and returns it as read.
func (c *twoFilers) seed(t *testing.T, p util.FullPath) *filer_pb.Entry {
t.Helper()
e := newFileEntry(string(p), 7)
e.Extended = map[string][]byte{"v": []byte("seed")}
if err := c.store.InsertEntry(context.Background(), e); err != nil {
t.Fatalf("seed %s: %v", p, err)
}
return e.ToProtoEntry()
}
// update sends addr an UpdateEntry that sets v to value only while v is still
// expected's.
func (c *twoFilers) update(addr pb.ServerAddress, p util.FullPath, expected *filer_pb.Entry, value string) error {
dir, _ := p.DirAndName()
next := proto.Clone(expected).(*filer_pb.Entry)
next.Extended = map[string][]byte{"v": []byte(value)}
return pb.WithFilerClient(false, 0, addr, c.dial, func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
Directory: dir,
Entry: next,
ExpectedExtended: map[string][]byte{"v": expected.Extended["v"]},
})
return err
})
}
func (c *twoFilers) stored(t *testing.T, p util.FullPath) string {
t.Helper()
e, err := c.store.FindEntry(context.Background(), p)
if err != nil {
t.Fatalf("read %s: %v", p, err)
}
return string(e.Extended["v"])
}
// A's ring names B as the write owner of the entry, while B, inside the
// cooling-off window of the change from {A} to {A, B}, names A. B must refuse
// A's forwarded conditional update rather than check it under its own lock,
// which another writer routed by B's ring would not take. A's ring has no
// window left to close, so A surfaces the refusal after its retries.
func TestForwardedConditionalUpdateRefusedByNonOwner(t *testing.T) {
slack := notWriteOwnerRetrySlack
notWriteOwnerRetrySlack = 200 * time.Millisecond
t.Cleanup(func() { notWriteOwnerRetrySlack = slack })
c := newTwoFilers(t)
setRing(c.fsA, 0, []pb.ServerAddress{c.addrA, c.addrB})
setRing(c.fsB, time.Minute, []pb.ServerAddress{c.addrA}, []pb.ServerAddress{c.addrA, c.addrB})
p := c.keyOwnedBy(t, c.addrB, c.addrA)
expected := c.seed(t, p)
err := c.update(c.addrA, p, expected, "via-A")
if !isNotWriteOwner(err) {
t.Fatalf("B applied or failed otherwise a forward its ring gives to A: %v", err)
}
if got := c.stored(t, p); got != "seed" {
t.Fatalf("a refused forward changed the entry to %q", got)
}
ua, ma := c.callsA.counts()
ub, mb := c.callsB.counts()
if ua != 1 || ma != 0 || ub < 2 || mb != ub {
t.Fatalf("updates received: A %d (forwarded %d), B %d (forwarded %d); want one on A, retried forwards on B", ua, ma, ub, mb)
}
}
// The same disagreement ends when B's window closes and B's ring names B too:
// A keeps re-routing the refused forward, B applies it once, and a second
// update with the same expectation fails its condition.
func TestForwardedConditionalUpdateAppliedOnceRingsAgree(t *testing.T) {
const window = time.Second
c := newTwoFilers(t)
setRing(c.fsA, window, []pb.ServerAddress{c.addrA, c.addrB})
setRing(c.fsB, window, []pb.ServerAddress{c.addrA}, []pb.ServerAddress{c.addrA, c.addrB})
p := c.keyOwnedBy(t, c.addrB, c.addrA)
expected := c.seed(t, p)
if err := c.update(c.addrA, p, expected, "via-A"); err != nil {
t.Fatalf("update via A once the rings agree: %v", err)
}
if got := c.stored(t, p); got != "via-A" {
t.Fatalf("stored %q, want via-A", got)
}
if ub, mb := c.callsB.counts(); ub < 2 || mb != ub {
t.Fatalf("B received %d updates (%d forwarded), want a refused forward before the applied one", ub, mb)
}
err := c.update(c.addrB, p, expected, "via-B")
if status.Code(err) != codes.FailedPrecondition {
t.Fatalf("the second update with the same expectation must fail its condition, got %v", err)
}
if got := c.stored(t, p); got != "via-A" {
t.Fatalf("stored %q after the losing update, want via-A", got)
}
}
@@ -856,11 +856,12 @@ func TestObjectTransactionRouteKeyOwnerAppliesLocally(t *testing.T) {
}
}
// A forwarded transaction (is_moved over a ring member's connection) on a
// filer whose ring names a different owner is refused, neither applied under
// this filer's lock nor forwarded again, so two filers that disagree on the
// owner during a ring change can neither both apply it nor loop.
func TestObjectTransactionIsMovedRefusedByNonOwner(t *testing.T) {
// A forwarded transaction (is_moved over a ring member's connection) applies
// locally even when the ring names a different owner: is_moved bounds
// forwarding to a single hop, so two filers that disagree on the owner during
// a ring change cannot loop. Without the marker this would attempt to dial
// the bogus owner instead of applying.
func TestObjectTransactionIsMovedSkipsForward(t *testing.T) {
self := pb.ServerAddress("localhost:1")
other := pb.ServerAddress("localhost:2")
fs, store := newTxnTestServer(t, map[string]*filer.Entry{
@@ -868,7 +869,7 @@ func TestObjectTransactionIsMovedRefusedByNonOwner(t *testing.T) {
})
withRing(fs, self, other) // ring owner is "other", not self
_, err := fs.ObjectTransaction(ringPeerCtx(), &filer_pb.ObjectTransactionRequest{
resp, err := fs.ObjectTransaction(ringPeerCtx(), &filer_pb.ObjectTransactionRequest{
LockKey: "/buckets/b/obj",
RouteKey: "s3.object.write:/buckets/b/obj",
IsMoved: true,
@@ -877,16 +878,18 @@ func TestObjectTransactionIsMovedRefusedByNonOwner(t *testing.T) {
SetExtended: map[string][]byte{"X-Amz-Meta-k": []byte("v")},
}},
})
if !isNotWriteOwner(err) {
t.Fatalf("a forwarded txn on a non-owner must be refused as not the write owner, got err=%v", err)
if err != nil || resp.Error != "" {
t.Fatalf("txn failed: err=%v resp=%q", err, resp.Error)
}
if _, set := store.entries["/buckets/b/obj"].Extended["X-Amz-Meta-k"]; set {
t.Errorf("a refused txn must not apply: %v", store.entries["/buckets/b/obj"].Extended)
if string(store.entries["/buckets/b/obj"].Extended["X-Amz-Meta-k"]) != "v" {
t.Errorf("forwarded txn should apply locally: %v", store.entries["/buckets/b/obj"].Extended)
}
}
// End-to-end forward hop: a non-owner filer dials the ring owner and the owner
// applies the transaction.
// applies the transaction. The owner's own ring points back at the (bogus)
// sender, so it would re-forward and fail to dial unless is_moved is set on the
// forwarded request — making this also assert that one-hop bound over the wire.
func TestObjectTransactionForwardsToOwner(t *testing.T) {
owner, ownerStore := newTxnTestServer(t, map[string]*filer.Entry{
"/buckets/b/obj": {FullPath: "/buckets/b/obj", Attr: filer.Attr{Inode: 1, Mode: 0644}},
@@ -902,7 +905,9 @@ func TestObjectTransactionForwardsToOwner(t *testing.T) {
ownerAddr := pb.NewServerAddressWithGrpcPort(lis.Addr().String(), port)
sender := pb.ServerAddress("127.0.0.1:1") // bogus: nothing listens here
withRing(owner, ownerAddr, ownerAddr)
// owner's ring points back at the sender; only is_moved keeps it from
// re-forwarding to (and failing to dial) that bogus address.
withRing(owner, ownerAddr, sender)
owner.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
srv := grpc.NewServer()
+8 -90
View File
@@ -11,7 +11,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
@@ -45,56 +44,18 @@ func (fs *FilerServer) writeOwner(key string) pb.ServerAddress {
// already committed — and an owner unreachable from here may be partitioned
// rather than down, still serving the key to everyone else. The ring hands the
// key on when the cooling-off window closes, so the outage is bounded.
//
// An owner that refuses the forward as not its own applied nothing: the two
// rings disagree. They converge once a ring update reaches the lagging filer or
// a cooling-off window closes, so the forward is re-routed by this filer's ring
// until shortly after its latest window ends, then the refusal surfaces. It
// also surfaces once this filer's ring names itself: a caller may have checked
// the entry before routing, and that check is stale after the wait.
func (fs *FilerServer) forwardToWriteOwner(ctx context.Context, key string, send func(owner pb.ServerAddress) error) (handled bool, err error) {
var firstRefusal time.Time
for wait := notWriteOwnerRetryWait; ; wait = min(2*wait, 10*notWriteOwnerRetryWait) {
owner := fs.writeOwner(key)
if owner == "" {
return err != nil, err
}
if err = send(owner); err == nil {
return true, nil
}
glog.V(1).InfofCtx(ctx, "route %s to owner %s: %v", key, owner, err)
if !isNotWriteOwner(err) {
return true, err
}
if firstRefusal.IsZero() {
firstRefusal = time.Now()
}
deadline := firstRefusal.Add(notWriteOwnerRetrySlack)
if end := fs.filer.Dlm.LockRing.PriorOwnerWindowEnd().Add(notWriteOwnerRetrySlack); end.After(deadline) {
deadline = end
}
if limit := firstRefusal.Add(notWriteOwnerRetryLimit); deadline.After(limit) {
deadline = limit
}
if time.Now().Add(wait).After(deadline) {
return true, err
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return true, err
case <-timer.C:
}
owner := fs.writeOwner(key)
if owner == "" {
return false, nil
}
if err := send(owner); err != nil {
glog.V(1).InfofCtx(ctx, "route %s to owner %s: %v", key, owner, err)
return true, err
}
return true, nil
}
var (
notWriteOwnerRetryWait = 50 * time.Millisecond
notWriteOwnerRetrySlack = time.Second
notWriteOwnerRetryLimit = 15 * time.Second
)
// entryRouteKey is the ring key for an entry's writes. It shares the S3
// gateway's namespace so an object's ObjectTransaction and its CreateEntry
// resolve to the same owner, and land on that filer's one per-path lock.
@@ -217,46 +178,3 @@ func (fs *FilerServer) checkMovedMarker(ctx context.Context, isMoved bool, owner
}
return status.Errorf(codes.PermissionDenied, "is_moved not sent by a ring member; the key's owner is %s", owner)
}
// notWriteOwnerReason marks a forwarded write refused by a filer whose own ring
// names another write owner for the key.
const notWriteOwnerReason = "NOT_WRITE_OWNER"
// checkForwardedWrite admits a routed write that arrived with is_moved only on
// the key's write owner in this filer's own ring. While two rings disagree, a
// forward applied here would be checked under a second filer's lock, and
// re-forwarding it could cycle, so it is refused. The refusal is Unavailable
// with a NOT_WRITE_OWNER reason, telling the sender nothing was applied.
func (fs *FilerServer) checkForwardedWrite(ctx context.Context, key string) error {
owner := fs.writeOwner(key)
if err := fs.checkMovedMarker(ctx, true, owner); err != nil {
return err
}
if owner == "" {
return nil
}
return notWriteOwnerError(key, owner)
}
func notWriteOwnerError(key string, owner pb.ServerAddress) error {
st := status.Newf(codes.Unavailable, "not the write owner of %s: %s is", key, owner)
withReason, err := st.WithDetails(&errdetails.ErrorInfo{Reason: notWriteOwnerReason, Domain: "seaweedfs"})
if err != nil {
return st.Err()
}
return withReason.Err()
}
// isNotWriteOwner reports whether err is checkForwardedWrite's refusal.
func isNotWriteOwner(err error) bool {
st, ok := util.ServerStatus(err)
if !ok || st.Code() != codes.Unavailable {
return false
}
for _, d := range st.Details() {
if info, ok := d.(*errdetails.ErrorInfo); ok && info.GetReason() == notWriteOwnerReason {
return true
}
}
return false
}
-110
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -107,112 +106,3 @@ func TestForwardToWriteOwnerNeverTriesASecondFiler(t *testing.T) {
t.Fatalf("expected exactly the prior owner, tried %v", tried)
}
}
// windowlessRouteServer is routeTestServer with no cooling-off window, so a
// ring change moves keys at once and the latest window has already closed.
func windowlessRouteServer(host pb.ServerAddress, servers []pb.ServerAddress) *FilerServer {
fs := routeTestServer(host)
fs.filer.Dlm.LockRing = lock_manager.NewLockRing(0)
fs.filer.Dlm.LockRing.SetSnapshot(servers, 1)
return fs
}
// A refused forward is not applied locally once this filer's ring names
// itself: the caller checked the entry before routing, and the wait made that
// check stale, so the refusal surfaces for the caller to retry.
func TestForwardToWriteOwnerSurfacesARefusalOnceOwned(t *testing.T) {
fs := windowlessRouteServer("f2:8888", routeSetA)
ring := fs.filer.Dlm.LockRing
key := firstKey(t, func(key string) bool { return ring.GetPrimary(key) != "f2:8888" })
var tried []pb.ServerAddress
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(owner pb.ServerAddress) error {
tried = append(tried, owner)
ring.SetSnapshot([]pb.ServerAddress{"f2:8888"}, 2)
return notWriteOwnerError(key, "f2:8888")
})
if !handled || !isNotWriteOwner(err) {
t.Fatalf("a refused forward must not fall back to a local apply, got handled=%v err=%v", handled, err)
}
if len(tried) != 1 {
t.Fatalf("expected one forward before the ring named this filer, tried %v", tried)
}
}
func TestForwardToWriteOwnerRetriesARefusal(t *testing.T) {
fs := windowlessRouteServer("f9:8888", routeSetA)
ring := fs.filer.Dlm.LockRing
key := firstKey(t, func(string) bool { return true })
var tried []pb.ServerAddress
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(owner pb.ServerAddress) error {
tried = append(tried, owner)
if len(tried) < 3 {
return notWriteOwnerError(key, "f9:8888")
}
return nil
})
if !handled || err != nil {
t.Fatalf("the owner accepting a retried forward must succeed, got handled=%v err=%v", handled, err)
}
if len(tried) != 3 || tried[0] != ring.GetPrimary(key) || tried[2] != ring.GetPrimary(key) {
t.Fatalf("expected three forwards to %s, tried %v", ring.GetPrimary(key), tried)
}
}
// Rings that stay apart (a filer cut off from the master) must not hold the
// write forever: the refusal surfaces once the retry budget is spent.
func TestForwardToWriteOwnerSurfacesAPersistentRefusal(t *testing.T) {
wait, slack := notWriteOwnerRetryWait, notWriteOwnerRetrySlack
notWriteOwnerRetryWait, notWriteOwnerRetrySlack = 5*time.Millisecond, 100*time.Millisecond
t.Cleanup(func() { notWriteOwnerRetryWait, notWriteOwnerRetrySlack = wait, slack })
fs := windowlessRouteServer("f9:8888", routeSetA)
key := firstKey(t, func(string) bool { return true })
tries := 0
start := time.Now()
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(pb.ServerAddress) error {
tries++
return notWriteOwnerError(key, "f9:8888")
})
if !handled || !isNotWriteOwner(err) {
t.Fatalf("a persistent refusal must surface, got handled=%v err=%v", handled, err)
}
if tries < 2 {
t.Fatalf("expected the refusal to be retried, tried %d times", tries)
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("retries outlived their budget: %v", elapsed)
}
}
// A ring change that arrives while a refused forward is being retried starts a
// new cooling-off window, and the retries wait for that one too.
func TestForwardToWriteOwnerWaitsOutANewWindow(t *testing.T) {
wait, slack := notWriteOwnerRetryWait, notWriteOwnerRetrySlack
notWriteOwnerRetryWait, notWriteOwnerRetrySlack = 5*time.Millisecond, 50*time.Millisecond
t.Cleanup(func() { notWriteOwnerRetryWait, notWriteOwnerRetrySlack = wait, slack })
const window = 100 * time.Millisecond
fs := routeTestServer("f9:8888")
ring := lock_manager.NewLockRing(window)
ring.SetSnapshot(routeSetA, 1)
fs.filer.Dlm.LockRing = ring
time.Sleep(window + 10*time.Millisecond)
key := firstKey(t, func(string) bool { return true })
tries := 0
start := time.Now()
handled, err := fs.forwardToWriteOwner(context.Background(), key, func(pb.ServerAddress) error {
tries++
if tries == 2 {
ring.SetSnapshot(routeSetB, 2)
}
if time.Since(start) < window {
return notWriteOwnerError(key, "f9:8888")
}
return nil
})
if !handled || err != nil {
t.Fatalf("a refusal must be retried through a window that opened during the retries, got handled=%v err=%v after %d tries", handled, err, tries)
}
}
@@ -3,7 +3,6 @@ package weed_server
import (
"bytes"
"context"
"crypto/fips140"
"errors"
"fmt"
"io"
@@ -65,7 +64,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
writeJsonError(w, r, http.StatusInternalServerError, err)
}
} else if reply != nil {
if len(md5bytes) > 0 && !fips140.Enforced() {
if len(md5bytes) > 0 {
md5InBase64 := util.Base64Encode(md5bytes)
w.Header().Set("Content-MD5", md5InBase64)
}
@@ -105,7 +104,8 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
return nil, nil, fmt.Errorf("read input: %w", err)
}
md5bytes = util.Md5(buf.Bytes())
if contentMd5Mismatch(md5bytes, r.Header.Get("Content-Md5")) {
headerMd5 := r.Header.Get("Content-Md5")
if headerMd5 != "" && !(util.Base64Encode(md5bytes) == headerMd5 || fmt.Sprintf("%x", md5bytes) == headerMd5) {
bufPool.Put(buf)
return nil, nil, errors.New(constants.ErrMsgBadDigest)
}
@@ -120,7 +120,8 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
}
md5bytes = md5Hash.Sum(nil)
if contentMd5Mismatch(md5bytes, r.Header.Get("Content-Md5")) {
headerMd5 := r.Header.Get("Content-Md5")
if headerMd5 != "" && !(util.Base64Encode(md5bytes) == headerMd5 || fmt.Sprintf("%x", md5bytes) == headerMd5) {
fs.filer.DeleteUncommittedChunks(ctx, fileChunks)
return nil, nil, errors.New(constants.ErrMsgBadDigest)
}
@@ -152,7 +153,8 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter
}
md5bytes = md5Hash.Sum(nil)
if contentMd5Mismatch(md5bytes, r.Header.Get("Content-Md5")) {
headerMd5 := r.Header.Get("Content-Md5")
if headerMd5 != "" && !(util.Base64Encode(md5bytes) == headerMd5 || fmt.Sprintf("%x", md5bytes) == headerMd5) {
fs.filer.DeleteUncommittedChunks(ctx, fileChunks)
return nil, nil, errors.New(constants.ErrMsgBadDigest)
}
@@ -713,13 +715,3 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
}
return filerResult, replyerr
}
// contentMd5Mismatch compares the body digest with the client's Content-MD5,
// base64 or hex. Under fips140=only the digest is not an MD5, so Content-MD5
// goes unchecked.
func contentMd5Mismatch(md5bytes []byte, headerMd5 string) bool {
if headerMd5 == "" || fips140.Enforced() {
return false
}
return util.Base64Encode(md5bytes) != headerMd5 && fmt.Sprintf("%x", md5bytes) != headerMd5
}
@@ -3,6 +3,7 @@ package weed_server
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"errors"
"fmt"
@@ -68,7 +69,7 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons
}
if int64(len(prefix)) < remainingInlineBudget {
inlineAppend := append([]byte{}, prefix...)
md5Hash = util.NewMd5()
md5Hash = md5.New()
_, _ = md5Hash.Write(inlineAppend)
if len(inlineAppend) > 0 {
stats.FilerHandlerCounter.WithLabelValues(stats.ContentSaveToFiler).Inc()
@@ -87,7 +88,7 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons
return nil, nil, 0, fmt.Errorf("read input: %w", readErr), nil
}
if len(prefix) == 0 {
return nil, util.NewMd5(), 0, nil, []byte{}
return nil, md5.New(), 0, nil, []byte{}
}
if chunkSize <= 0 {
return nil, nil, 0, fmt.Errorf("invalid chunk size %d for inline append promotion", chunkSize), nil
@@ -102,7 +103,7 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons
func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, chunkSize int32, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
md5Hash = util.NewMd5()
md5Hash = md5.New()
chunkOffset = startOffset
var partReader = io.NopCloser(io.TeeReader(reader, md5Hash))
@@ -271,7 +272,7 @@ func (fs *FilerServer) dataToChunkWithSSE(ctx context.Context, r *http.Request,
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
return uploadErr
}
chunkMd5 := util.Md5(data)
chunkMd5 := md5.Sum(data)
chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
// upload the chunk to the volume server
uploadResult, uploadErr, _ = fs.doUpload(ctx, urlLocation, dataReader, fileName, contentType, nil, auth, chunkMd5B64)
-4
View File
@@ -2,7 +2,6 @@ package postgres
import (
"bufio"
"crypto/fips140"
"crypto/md5"
"crypto/rand"
"crypto/tls"
@@ -163,9 +162,6 @@ type FieldDescription struct {
// NewPostgreSQLServer creates a new PostgreSQL protocol server
func NewPostgreSQLServer(config *PostgreSQLServerConfig, masterAddr string) (*PostgreSQLServer, error) {
if config.AuthMethod == AuthMD5 && fips140.Enforced() {
return nil, fmt.Errorf("md5 authentication uses MD5, which GODEBUG=fips140=only refuses")
}
if config.Port <= 0 {
config.Port = DEFAULT_POSTGRES_PORT
}
@@ -3,7 +3,6 @@ package weed_server
import (
"bytes"
"context"
"crypto/fips140"
"crypto/md5"
"encoding/base64"
"errors"
@@ -33,9 +32,6 @@ func (vs *VolumeServer) replicaContentMd5(n *needle.Needle, contentMd5 string) s
if !vs.FixJpgOrientation || contentMd5 == "" || !bytes.HasPrefix(n.Data, jpegStart) {
return contentMd5
}
if fips140.Enforced() {
return ""
}
if sum := md5.Sum(n.Data); base64.StdEncoding.EncodeToString(sum[:]) != contentMd5 {
return ""
}
+2 -1
View File
@@ -3,6 +3,7 @@ package sftpd
import (
"context"
"crypto/md5"
"encoding/json"
"errors"
"fmt"
@@ -602,7 +603,7 @@ func (fs *SftpServer) putFile(filepath string, reader io.Reader, user *user.User
uploadUrl = normalizedUrl
// Compute MD5 while uploading
hash := util.NewMd5()
hash := md5.New()
body := io.TeeReader(reader, hash)
// We can skip ContentLength if unknown (chunked transfer encoding)
+8 -28
View File
@@ -277,7 +277,14 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE
return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
}
maskRemoteConfSecrets(conf)
// change secret key to stars
conf.S3SecretKey = strings.Repeat("*", len(conf.S3SecretKey))
conf.AliyunSecretKey = strings.Repeat("*", len(conf.AliyunSecretKey))
conf.BaiduAccessKey = strings.Repeat("*", len(conf.BaiduAccessKey))
conf.FilebaseSecretKey = strings.Repeat("*", len(conf.FilebaseSecretKey))
conf.StorjSecretKey = strings.Repeat("*", len(conf.StorjSecretKey))
conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
return filer.ProtoToText(writer, conf)
@@ -318,30 +325,3 @@ func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, write
return nil
}
// maskRemoteConfSecrets replaces every secret of a remote configuration with
// stars of the same length before it is listed: the listing still shows
// whether a secret is set, never its value.
func maskRemoteConfSecrets(conf *remote_pb.RemoteConf) {
for _, secret := range []*string{
&conf.S3SecretKey,
&conf.AzureAccountKey,
&conf.BackblazeApplicationKey,
&conf.AliyunSecretKey,
&conf.TencentSecretKey,
&conf.BaiduAccessKey,
&conf.BaiduSecretKey,
&conf.WasabiSecretKey,
&conf.FilebaseSecretKey,
&conf.StorjSecretKey,
&conf.ContaboSecretKey,
} {
*secret = strings.Repeat("*", len(*secret))
}
// The gcs credentials are a file path (remote.configure's
// -gcs.appCredentialsFile) or the service account JSON itself, private key
// included: a path is listed, inline JSON is masked.
if strings.HasPrefix(strings.TrimSpace(conf.GcsGoogleApplicationCredentials), "{") {
conf.GcsGoogleApplicationCredentials = strings.Repeat("*", len(conf.GcsGoogleApplicationCredentials))
}
}
@@ -1,69 +0,0 @@
package shell
import (
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"google.golang.org/protobuf/proto"
)
func TestMaskRemoteConfSecrets(t *testing.T) {
conf := &remote_pb.RemoteConf{
Type: "azure",
Name: "r1",
S3AccessKey: "AKID",
S3SecretKey: "s3-secret",
AzureAccountName: "account",
AzureAccountKey: "azure-account-key==",
BackblazeApplicationKey: "b2-secret",
AliyunSecretKey: "aliyun-secret",
TencentSecretKey: "tencent-secret",
BaiduAccessKey: "baidu-ak",
BaiduSecretKey: "baidu-secret",
WasabiSecretKey: "wasabi-secret",
FilebaseSecretKey: "filebase-secret",
StorjSecretKey: "storj-secret",
ContaboSecretKey: "contabo-secret",
}
original := proto.Clone(conf).(*remote_pb.RemoteConf)
maskRemoteConfSecrets(conf)
for name, pair := range map[string][2]string{
"s3": {original.S3SecretKey, conf.S3SecretKey},
"azure": {original.AzureAccountKey, conf.AzureAccountKey},
"backblaze": {original.BackblazeApplicationKey, conf.BackblazeApplicationKey},
"aliyun": {original.AliyunSecretKey, conf.AliyunSecretKey},
"tencent": {original.TencentSecretKey, conf.TencentSecretKey},
"baidu ak": {original.BaiduAccessKey, conf.BaiduAccessKey},
"baidu sk": {original.BaiduSecretKey, conf.BaiduSecretKey},
"wasabi": {original.WasabiSecretKey, conf.WasabiSecretKey},
"filebase": {original.FilebaseSecretKey, conf.FilebaseSecretKey},
"storj": {original.StorjSecretKey, conf.StorjSecretKey},
"contabo": {original.ContaboSecretKey, conf.ContaboSecretKey},
} {
if want := strings.Repeat("*", len(pair[0])); pair[1] != want {
t.Errorf("%s secret masked as %q, want %q", name, pair[1], want)
}
}
if conf.AzureAccountName != "account" || conf.S3AccessKey != "AKID" {
t.Errorf("non-secret fields changed: %q %q", conf.AzureAccountName, conf.S3AccessKey)
}
empty := &remote_pb.RemoteConf{Type: "azure"}
maskRemoteConfSecrets(empty)
if empty.AzureAccountKey != "" {
t.Errorf("an unset key must stay empty (Entra ID), got %q", empty.AzureAccountKey)
}
inline := ` {"type":"service_account","private_key":"-----BEGIN PRIVATE KEY-----"}`
gcs := &remote_pb.RemoteConf{Type: "gcs", GcsGoogleApplicationCredentials: inline}
maskRemoteConfSecrets(gcs)
if want := strings.Repeat("*", len(inline)); gcs.GcsGoogleApplicationCredentials != want {
t.Errorf("inline gcs credentials masked as %q, want %q", gcs.GcsGoogleApplicationCredentials, want)
}
path := &remote_pb.RemoteConf{Type: "gcs", GcsGoogleApplicationCredentials: "/etc/seaweedfs/gcs.json"}
maskRemoteConfSecrets(path)
if path.GcsGoogleApplicationCredentials != "/etc/seaweedfs/gcs.json" {
t.Errorf("a gcs credentials path must stay listed, got %q", path.GcsGoogleApplicationCredentials)
}
}
+1 -5
View File
@@ -2,7 +2,6 @@ package needle
import (
"bytes"
"crypto/fips140"
"crypto/md5"
"encoding/base64"
"fmt"
@@ -123,10 +122,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
if expectedChecksum == "" {
expectedChecksum = pu.ContentMd5
}
if expectedChecksum != "" && fips140.Enforced() {
// fips140=only refuses MD5: keep the client's digest unchecked.
pu.ContentMd5 = expectedChecksum
} else if expectedChecksum != "" {
if expectedChecksum != "" {
h := md5.New()
h.Write(pu.UncompressedData)
pu.ContentMd5 = base64.StdEncoding.EncodeToString(h.Sum(nil))
+4 -3
View File
@@ -2,6 +2,7 @@ package util
import (
"bytes"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"fmt"
@@ -88,7 +89,7 @@ func Uint8toBytes(b []byte, v uint8) {
// returns a 64 bit big int
func HashStringToLong(dir string) (v int64) {
h := NewMd5()
h := md5.New()
io.WriteString(h, dir)
b := h.Sum(nil)
@@ -113,7 +114,7 @@ func HashStringToLong(dir string) (v int64) {
}
func HashToInt32(data []byte) (v int32) {
h := NewMd5()
h := md5.New()
h.Write(data)
b := h.Sum(nil)
@@ -134,7 +135,7 @@ func Base64Encode(data []byte) string {
}
func Md5(data []byte) []byte {
hash := NewMd5()
hash := md5.New()
hash.Write(data)
return hash.Sum(nil)
}
-44
View File
@@ -1,44 +0,0 @@
package util
import (
"crypto/fips140"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"hash"
)
var fips140Enforced = fips140.Enforced
// NewMd5 returns an MD5 hash. GODEBUG=fips140=only refuses MD5, so there it
// returns SHA-256 cut to MD5's size: digests keep their length but differ.
func NewMd5() hash.Hash {
if fips140Enforced() {
return NewTruncatedSha256(md5.Size)
}
return md5.New()
}
// NewSha1 is NewMd5 for SHA-1.
func NewSha1() hash.Hash {
if fips140Enforced() {
return NewTruncatedSha256(sha1.Size)
}
return sha1.New()
}
// NewTruncatedSha256 returns SHA-256 keeping the first size bytes of the digest.
func NewTruncatedSha256(size int) hash.Hash {
return &truncatedHash{Hash: sha256.New(), size: size}
}
type truncatedHash struct {
hash.Hash
size int
}
func (h *truncatedHash) Size() int { return h.size }
func (h *truncatedHash) Sum(b []byte) []byte {
return append(b, h.Hash.Sum(nil)[:h.size]...)
}
-59
View File
@@ -1,59 +0,0 @@
package util
import (
"bytes"
"crypto/fips140"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"testing"
)
func setFIPS140Enforced(t *testing.T, enforced bool) {
t.Helper()
saved := fips140Enforced
fips140Enforced = func() bool { return enforced }
t.Cleanup(func() { fips140Enforced = saved })
}
func TestNewMd5KeepsMd5WhenNotEnforced(t *testing.T) {
if fips140.Enforced() {
t.Skip("GODEBUG=fips140=only refuses MD5 and SHA-1")
}
setFIPS140Enforced(t, false)
data := []byte("hello")
want := md5.Sum(data)
if got := Md5(data); !bytes.Equal(got, want[:]) {
t.Fatalf("Md5 = %x, want %x", got, want)
}
h := NewSha1()
h.Write(data)
wantSha1 := sha1.Sum(data)
if got := h.Sum(nil); !bytes.Equal(got, wantSha1[:]) {
t.Fatalf("NewSha1 = %x, want %x", got, wantSha1)
}
}
func TestNewMd5UsesTruncatedSha256WhenEnforced(t *testing.T) {
setFIPS140Enforced(t, true)
data := []byte("hello")
full := sha256.Sum256(data)
got := Md5(data)
if !bytes.Equal(got, full[:md5.Size]) {
t.Fatalf("Md5 = %x, want %x", got, full[:md5.Size])
}
h := NewSha1()
if h.Size() != sha1.Size {
t.Fatalf("NewSha1 size = %d, want %d", h.Size(), sha1.Size)
}
h.Write(data)
if got := h.Sum([]byte{0xff}); !bytes.Equal(got, append([]byte{0xff}, full[:sha1.Size]...)) {
t.Fatalf("NewSha1 Sum = %x", got)
}
if HashStringToLong("/a/b") == 0 || HashToInt32([]byte("k")) == 0 {
t.Fatalf("hashes should be non-zero")
}
}