mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* s3: keep serving for -preStopSeconds after SIGTERM On SIGTERM the gateway closed its listeners at once, so requests a load balancer or Kubernetes Service still routed to it failed until the endpoint was removed. With -preStopSeconds set, the gateway makes /status, /healthz and /readyz return 503, keeps serving for that many seconds, then runs the existing graceful shutdown. 0 (the default) keeps the old behavior. The flag is registered for weed s3 and as -s3.preStopSeconds for weed filer, weed server and weed mini. A cancelled weed mini cluster context ends the wait early. The Helm chart passes s3.preStopSeconds and filer.s3.preStopSeconds through. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: assert the preStopSeconds chart render under set -e, drop timing from the probe test Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: wait before weed mini cancels its clients; size grace periods in the chart weed mini's interrupt hook runs before the S3 one and cancels the S3 gateway's shutdownCtx, so the pre-stop wait never happened there. The wait is now also registered with beforeMiniClientsShutdown, which runs before that cancel, and a sync.OnceFunc keeps the interrupt hook from waiting a second time. Outside weed mini the registration is a no-op. Helm: the s3 deployment fails to render when terminationGracePeriodSeconds is below preStopSeconds + 15. The filer statefulset and the all-in-one deployment, whose grace period is fixed at 60s, add preStopSeconds to it. allInOne.s3.preStopSeconds is passed through, inheriting s3.preStopSeconds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: keep 64-bit counters aligned on 386; let allInOne.s3.preStopSeconds=0 disable the wait draining sat ahead of inFlightDataSize and inFlightUploads in S3ApiServer and shifted them off 8-byte alignment on 32-bit builds, so the atomic upload-limit counters panicked there. It now follows them. Helm: allInOne.s3.preStopSeconds inherits s3.preStopSeconds only when null; default treated an explicit 0 as unset. The s3 values and the grace period error now say a lifecycle preStop hook's time comes on top. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grace: add OnInterruptFirst for hooks that must run before any shutdown Interrupt hooks run in reverse registration order, so a hook cannot run ahead of a service that registers later, such as a volume server that is still loading its volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: run the -preStopSeconds wait before every other interrupt hook Under weed server the volume server registers its hook once its volumes are loaded, often after the S3 gateway. Its hook then ran first and shut the volume server down, so the gateway spent the whole wait serving requests that could not read or write objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
946 lines
38 KiB
Go
946 lines
38 KiB
Go
package command
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/gorilla/mux"
|
|
"google.golang.org/grpc/reflection"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/iceberg"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
var (
|
|
s3StandaloneOptions S3Options
|
|
)
|
|
|
|
// S3Options holds CLI flags for the S3 gateway.
|
|
// Flags are registered in multiple commands: s3.go (standalone), server.go, filer.go, and mini.go.
|
|
// When adding a new field, update all four flag registration sites.
|
|
type S3Options struct {
|
|
filer *string
|
|
ip *string
|
|
bindIp *string
|
|
listenOnLocalhost *bool
|
|
port *int
|
|
portHttps *int
|
|
portGrpc *int
|
|
portIceberg *int
|
|
portLance *int
|
|
icebergCredentialRole *string
|
|
icebergCredentialDuration *int
|
|
config *string
|
|
iamConfig *string
|
|
domainName *string
|
|
allowedOrigins *string
|
|
tlsPrivateKey *string
|
|
tlsCertificate *string
|
|
tlsCACertificate *string
|
|
tlsVerifyClientCert *bool
|
|
metricsHttpPort *int
|
|
metricsHttpIp *string
|
|
allowDeleteBucketNotEmpty *bool
|
|
autoCreateBucket *bool
|
|
auditLogConfig *string
|
|
localFilerSocket *string
|
|
dataCenter *string
|
|
localSocket *string
|
|
idleTimeout *int
|
|
concurrentUploadLimitMB *int
|
|
concurrentFileUploadLimit *int
|
|
inflightUploadDataTimeout *time.Duration
|
|
enableIam *bool
|
|
iamReadOnly *bool
|
|
debug *bool
|
|
debugPort *int
|
|
cipher *bool
|
|
externalUrl *string
|
|
defaultFileMode *string
|
|
cacheSizeMB *int64
|
|
readerCacheSizeMB *int64
|
|
|
|
allowUntrustedRemoteEndpoints *bool
|
|
allowBackslashInKey *bool
|
|
iamSelfServiceRequiresGrant *bool
|
|
preStopSeconds *int
|
|
// shutdownCtx, when non-nil, tells startS3Server to gracefully shut down
|
|
// its HTTP/gRPC servers once the ctx is cancelled, in addition to on
|
|
// interrupt. Used by weed mini to orchestrate an ordered shutdown.
|
|
shutdownCtx context.Context
|
|
|
|
// icebergBound/lanceBound record whether the auxiliary catalog listeners
|
|
// bound their ports. The aux binds happen before the S3 listener serves,
|
|
// so once the S3 port answers these are final; mini reads them to probe
|
|
// and advertise only catalogs that are actually running instead of
|
|
// whatever process may hold the configured port.
|
|
icebergBound atomic.Bool
|
|
lanceBound atomic.Bool
|
|
lanceServer *lance.Server
|
|
}
|
|
|
|
func init() {
|
|
cmdS3.Run = runS3 // break init cycle
|
|
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
|
|
s3StandaloneOptions.ip = cmdS3.Flag.String("ip", "", "ip address advertised to the cluster. If empty, default to -ip.bind, or the auto-detected address.")
|
|
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to 0.0.0.0.")
|
|
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
|
|
s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
|
|
s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
|
|
s3StandaloneOptions.portIceberg = cmdS3.Flag.Int("port.iceberg", 8181, "Iceberg REST Catalog server listen port (0 to disable)")
|
|
s3StandaloneOptions.portLance = cmdS3.Flag.Int("port.lance", 9101, "Lance Namespace server listen port (0 to disable); credential vending uses -iceberg.credentialRole")
|
|
s3StandaloneOptions.icebergCredentialRole = cmdS3.Flag.String("iceberg.credentialRole", "", "IAM role ARN the Iceberg catalog assumes to vend table-scoped credentials (empty disables vending)")
|
|
s3StandaloneOptions.icebergCredentialDuration = cmdS3.Flag.Int("iceberg.credentialDurationSeconds", 3600, "lifetime of credentials vended by the Iceberg catalog")
|
|
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
|
s3StandaloneOptions.allowedOrigins = cmdS3.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
|
|
s3StandaloneOptions.dataCenter = cmdS3.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
|
s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
|
|
s3StandaloneOptions.iamConfig = cmdS3.Flag.String("iam.config", "", "path to the advanced IAM config file")
|
|
s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
|
|
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
|
|
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
|
|
s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file")
|
|
s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate")
|
|
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
|
s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
|
cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
|
|
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
|
s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
|
s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
|
|
s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
|
|
s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds")
|
|
s3StandaloneOptions.concurrentUploadLimitMB = cmdS3.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
|
s3StandaloneOptions.concurrentFileUploadLimit = cmdS3.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited")
|
|
s3StandaloneOptions.inflightUploadDataTimeout = cmdS3.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "how long an upload waits for a slot under the concurrent upload limits before it is rejected with SlowDown, 0 means no limit")
|
|
s3StandaloneOptions.enableIam = cmdS3.Flag.Bool("iam", true, "enable embedded IAM API on the same port")
|
|
s3StandaloneOptions.iamReadOnly = cmdS3.Flag.Bool("iam.readOnly", true, "disable IAM write operations on this server")
|
|
s3StandaloneOptions.debug = cmdS3.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
|
|
s3StandaloneOptions.debugPort = cmdS3.Flag.Int("debug.port", 6060, "http port for debugging")
|
|
s3StandaloneOptions.cipher = cmdS3.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers; on whenever the filer's -encryptVolumeData is")
|
|
s3StandaloneOptions.externalUrl = cmdS3.Flag.String("externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Advertised to Iceberg and Lance clients, and tried first when verifying S3 signatures behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
|
|
s3StandaloneOptions.defaultFileMode = cmdS3.Flag.String("defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
|
|
s3StandaloneOptions.cacheSizeMB = cmdS3.Flag.Int64("cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
|
|
s3StandaloneOptions.readerCacheSizeMB = cmdS3.Flag.Int64("readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
|
|
s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
|
s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage)
|
|
s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
|
s3StandaloneOptions.preStopSeconds = cmdS3.Flag.Int("preStopSeconds", 0, s3PreStopSecondsUsage)
|
|
}
|
|
|
|
var cmdS3 = &Command{
|
|
UsageLine: "s3 [-port=8333] [-filer=<ip:port>[,<ip:port>]...] [-config=</path/to/config.json>]",
|
|
Short: "start a s3 API compatible server that is backed by filer(s)",
|
|
Long: `start a s3 API compatible server that is backed by filer(s).
|
|
|
|
Multiple filer addresses can be specified for high availability, separated by commas.
|
|
The S3 server will automatically failover between filers if one becomes unavailable.
|
|
|
|
By default, you can use any access key and secret key to access the S3 APIs.
|
|
To enable credential based access, create a config.json file similar to this:
|
|
|
|
{
|
|
"identities": [
|
|
{
|
|
"name": "anonymous",
|
|
"actions": [
|
|
"Read"
|
|
]
|
|
},
|
|
{
|
|
"name": "some_admin_user",
|
|
"credentials": [
|
|
{
|
|
"accessKey": "some_access_key1",
|
|
"secretKey": "some_secret_key1"
|
|
}
|
|
],
|
|
"actions": [
|
|
"Admin",
|
|
"Read",
|
|
"List",
|
|
"Tagging",
|
|
"Write"
|
|
]
|
|
},
|
|
{
|
|
"name": "some_read_only_user",
|
|
"credentials": [
|
|
{
|
|
"accessKey": "some_access_key2",
|
|
"secretKey": "some_secret_key2"
|
|
}
|
|
],
|
|
"actions": [
|
|
"Read"
|
|
]
|
|
},
|
|
{
|
|
"name": "some_normal_user",
|
|
"credentials": [
|
|
{
|
|
"accessKey": "some_access_key3",
|
|
"secretKey": "some_secret_key3"
|
|
}
|
|
],
|
|
"actions": [
|
|
"Read",
|
|
"List",
|
|
"Tagging",
|
|
"Write"
|
|
]
|
|
},
|
|
{
|
|
"name": "user_limited_to_bucket1",
|
|
"credentials": [
|
|
{
|
|
"accessKey": "some_access_key4",
|
|
"secretKey": "some_secret_key4"
|
|
}
|
|
],
|
|
"actions": [
|
|
"Read:bucket1",
|
|
"List:bucket1",
|
|
"Tagging:bucket1",
|
|
"Write:bucket1"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
|
|
Alternatively, you can use environment variables as fallback admin credentials:
|
|
|
|
AWS_ACCESS_KEY_ID=your_access_key AWS_SECRET_ACCESS_KEY=your_secret_key weed s3
|
|
|
|
Environment variables are only used when no S3 configuration file is provided
|
|
and no configuration is available from the filer. This provides a simple way
|
|
to get started without requiring configuration files.
|
|
|
|
`,
|
|
}
|
|
|
|
func runS3(cmd *Command, args []string) bool {
|
|
if *s3StandaloneOptions.debug {
|
|
grace.StartDebugServer(*s3StandaloneOptions.debugPort)
|
|
}
|
|
|
|
s3StandaloneOptions.resolvePaths()
|
|
util.LoadSecurityConfiguration()
|
|
|
|
switch {
|
|
case *s3StandaloneOptions.metricsHttpIp != "":
|
|
// noting to do, use s3StandaloneOptions.metricsHttpIp
|
|
case *s3StandaloneOptions.bindIp != "":
|
|
*s3StandaloneOptions.metricsHttpIp = *s3StandaloneOptions.bindIp
|
|
}
|
|
go stats_collect.StartMetricsServer(*s3StandaloneOptions.metricsHttpIp, *s3StandaloneOptions.metricsHttpPort)
|
|
|
|
return s3StandaloneOptions.startS3Server()
|
|
|
|
}
|
|
|
|
// resolveExternalUrl returns the external URL from the flag or falls back to the S3_EXTERNAL_URL env var.
|
|
func (s3opt *S3Options) resolveExternalUrl() string {
|
|
if s3opt.externalUrl != nil && *s3opt.externalUrl != "" {
|
|
return *s3opt.externalUrl
|
|
}
|
|
return os.Getenv("S3_EXTERNAL_URL")
|
|
}
|
|
|
|
func (s3opt *S3Options) parseDefaultFileMode() (uint32, error) {
|
|
if s3opt.defaultFileMode == nil || *s3opt.defaultFileMode == "" {
|
|
return 0, nil
|
|
}
|
|
mode, err := strconv.ParseUint(*s3opt.defaultFileMode, 8, 32)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("invalid defaultFileMode %q: %v", *s3opt.defaultFileMode, err)
|
|
}
|
|
return uint32(mode), nil
|
|
}
|
|
|
|
// resolvePaths expands "~" in every user-supplied path flag so callers
|
|
// that share these pointers (e.g. server.go propagating s3Options.config
|
|
// to filerOptions.s3ConfigFile before startS3Server runs) see resolved
|
|
// values. Idempotent — safe to call from any entry point.
|
|
func (s3opt *S3Options) resolvePaths() {
|
|
*s3opt.config = util.ResolvePath(*s3opt.config)
|
|
*s3opt.iamConfig = util.ResolvePath(*s3opt.iamConfig)
|
|
*s3opt.tlsCertificate = util.ResolvePath(*s3opt.tlsCertificate)
|
|
*s3opt.tlsPrivateKey = util.ResolvePath(*s3opt.tlsPrivateKey)
|
|
*s3opt.tlsCACertificate = util.ResolvePath(*s3opt.tlsCACertificate)
|
|
*s3opt.auditLogConfig = util.ResolvePath(*s3opt.auditLogConfig)
|
|
}
|
|
|
|
func (s3opt *S3Options) startS3Server() bool {
|
|
|
|
// Before the first filer dial below; gRPC caches conns and binds at dial time.
|
|
util.SetOutboundLocalIP(*s3opt.bindIp)
|
|
|
|
filerAddresses := pb.ServerAddresses(*s3opt.filer).ToAddresses()
|
|
|
|
filerBucketsPath := "/buckets"
|
|
filerGroup := ""
|
|
var masterAddresses []pb.ServerAddress
|
|
|
|
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
|
|
|
// metrics read from the filer
|
|
var metricsAddress string
|
|
var metricsIntervalSec int
|
|
|
|
// chunk size for S3 uploads, read from the filer's -maxMB
|
|
var filerMaxMB int32
|
|
// the filer's -encryptVolumeData, which turns on chunk encryption here too
|
|
var filerCipher bool
|
|
|
|
for {
|
|
err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
|
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
|
if err != nil {
|
|
return fmt.Errorf("get filer configuration: %v", err)
|
|
}
|
|
filerBucketsPath = resp.DirBuckets
|
|
filerGroup = resp.FilerGroup
|
|
// Get master addresses for filer discovery
|
|
masterAddresses = pb.ServerAddresses(strings.Join(resp.Masters, ",")).ToAddresses()
|
|
metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
|
|
filerMaxMB = int32(resp.MaxMb)
|
|
filerCipher = resp.Cipher
|
|
glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
|
|
glog.V(0).Infof("S3 read filer maxMB: %d", filerMaxMB)
|
|
if len(masterAddresses) > 0 {
|
|
glog.V(0).Infof("S3 read master addresses for discovery: %v", masterAddresses)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
glog.V(2).Infof("wait to connect to filers %v grpc address", filerAddresses)
|
|
time.Sleep(time.Second)
|
|
} else {
|
|
glog.V(0).Infof("connected to filers %v", filerAddresses)
|
|
break
|
|
}
|
|
}
|
|
|
|
go stats_collect.LoopPushingMetric("s3", stats_collect.SourceName(uint32(*s3opt.port)), metricsAddress, metricsIntervalSec)
|
|
|
|
router := mux.NewRouter().SkipClean(true)
|
|
var localFilerSocket string
|
|
// The filer only listens on its local socket outside Windows.
|
|
if s3opt.localFilerSocket != nil && runtime.GOOS != "windows" {
|
|
localFilerSocket = *s3opt.localFilerSocket
|
|
}
|
|
var s3ApiServer *s3api.S3ApiServer
|
|
var s3ApiServer_err error
|
|
|
|
// Create S3 server with optional advanced IAM integration
|
|
var iamConfigPath string
|
|
if s3opt.iamConfig != nil && *s3opt.iamConfig != "" {
|
|
iamConfigPath = *s3opt.iamConfig
|
|
glog.V(0).Infof("Starting S3 API Server with advanced IAM integration")
|
|
} else {
|
|
glog.V(0).Infof("Starting S3 API Server with standard IAM")
|
|
}
|
|
|
|
if *s3opt.portGrpc == 0 {
|
|
*s3opt.portGrpc = 10000 + *s3opt.port
|
|
}
|
|
// The loopback companion listener is only kept while -ip.bind is unset.
|
|
// An explicit -ip.bind pins each port to exactly that address.
|
|
listenOnLocalhost := *s3opt.bindIp == ""
|
|
if s3opt.listenOnLocalhost != nil {
|
|
listenOnLocalhost = *s3opt.listenOnLocalhost
|
|
}
|
|
if *s3opt.bindIp == "" {
|
|
*s3opt.bindIp = "0.0.0.0"
|
|
}
|
|
|
|
defaultFileMode, fileModeErr := s3opt.parseDefaultFileMode()
|
|
if fileModeErr != nil {
|
|
glog.Fatalf("S3 API Server startup error: %v", fileModeErr)
|
|
}
|
|
|
|
var readerCacheSizeMB int64
|
|
if s3opt.readerCacheSizeMB != nil {
|
|
readerCacheSizeMB = *s3opt.readerCacheSizeMB
|
|
}
|
|
|
|
s3ApiServer, s3ApiServer_err = s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
|
|
Filers: filerAddresses,
|
|
Masters: masterAddresses,
|
|
Port: *s3opt.port,
|
|
Config: *s3opt.config,
|
|
DomainName: *s3opt.domainName,
|
|
AllowedOrigins: strings.Split(*s3opt.allowedOrigins, ","),
|
|
BucketsPath: filerBucketsPath,
|
|
GrpcDialOption: grpcDialOption,
|
|
AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
|
|
AutoCreateBucket: *s3opt.autoCreateBucket,
|
|
LocalFilerSocket: localFilerSocket,
|
|
DataCenter: *s3opt.dataCenter,
|
|
FilerGroup: filerGroup,
|
|
IamConfig: iamConfigPath, // Advanced IAM config (optional)
|
|
ConcurrentUploadLimit: int64(*s3opt.concurrentUploadLimitMB) * 1024 * 1024,
|
|
ConcurrentFileUploadLimit: int64(*s3opt.concurrentFileUploadLimit),
|
|
InflightUploadDataTimeout: *s3opt.inflightUploadDataTimeout,
|
|
EnableIam: *s3opt.enableIam, // Embedded IAM API (enabled by default)
|
|
IamReadOnly: *s3opt.iamReadOnly,
|
|
Cipher: *s3opt.cipher || filerCipher, // encrypt data on volume servers
|
|
Ip: *s3opt.ip,
|
|
BindIp: *s3opt.bindIp,
|
|
GrpcPort: *s3opt.portGrpc,
|
|
ExternalUrl: s3opt.resolveExternalUrl(),
|
|
DefaultFileMode: defaultFileMode,
|
|
CacheSizeMB: *s3opt.cacheSizeMB,
|
|
ReaderCacheSizeMB: readerCacheSizeMB,
|
|
MaxMB: filerMaxMB,
|
|
|
|
AllowUntrustedRemoteEndpoints: *s3opt.allowUntrustedRemoteEndpoints,
|
|
AllowBackslashInKey: *s3opt.allowBackslashInKey,
|
|
IamSelfServiceRequiresGrant: *s3opt.iamSelfServiceRequiresGrant,
|
|
})
|
|
if s3ApiServer_err != nil {
|
|
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
|
|
}
|
|
|
|
var httpServers []*http.Server
|
|
|
|
// Once, before either catalog serves a request that reads it.
|
|
s3opt.setCredentialRole(s3ApiServer)
|
|
|
|
// Start Iceberg REST Catalog server if enabled
|
|
if *s3opt.portIceberg > 0 {
|
|
if httpS := s3opt.startIcebergServer(s3ApiServer, listenOnLocalhost); httpS != nil {
|
|
httpServers = append(httpServers, httpS)
|
|
}
|
|
}
|
|
|
|
// The S3 Vectors API resolves its tables through the catalog in process,
|
|
// whether or not the catalog's own port is served.
|
|
s3opt.lanceServer = s3opt.newLanceServer(s3ApiServer)
|
|
if s3opt.portLance != nil && *s3opt.portLance > 0 {
|
|
if httpS := s3opt.startLanceServer(listenOnLocalhost); httpS != nil {
|
|
httpServers = append(httpServers, httpS)
|
|
}
|
|
}
|
|
|
|
// The Vectors API rides the main -port listener whenever a weed-vector
|
|
// backend is reachable — /s3vectors/ in the SigV4 credential scope picks
|
|
// the vectors router.
|
|
vectorsHandler := s3Handler(router, s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses, filerGroup))
|
|
|
|
if runtime.GOOS != "windows" {
|
|
localSocket := *s3opt.localSocket
|
|
explicit := localSocket != ""
|
|
if !explicit {
|
|
localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port)
|
|
}
|
|
if s3SocketListener := pb.ListenOnLocalSocket(localSocket); s3SocketListener != nil {
|
|
s3SocketListener = &util.Listener{Listener: s3SocketListener, Timeout: idleTimeoutSeconds(s3opt.idleTimeout)}
|
|
socketServer := newHttpServer(vectorsHandler, nil)
|
|
httpServers = append(httpServers, socketServer)
|
|
go func() {
|
|
if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("Failed to start S3 http server: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
|
|
s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(
|
|
*s3opt.bindIp, *s3opt.port, idleTimeoutSeconds(s3opt.idleTimeout), listenOnLocalhost)
|
|
if err != nil {
|
|
glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
|
|
}
|
|
|
|
if len(*s3opt.auditLogConfig) > 0 {
|
|
s3err.InitAuditLog(*s3opt.auditLogConfig)
|
|
if s3err.Logger != nil {
|
|
defer s3err.Logger.Close()
|
|
}
|
|
}
|
|
|
|
// starting grpc server
|
|
grpcPort := *s3opt.portGrpc
|
|
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0, listenOnLocalhost)
|
|
if err != nil {
|
|
glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
|
|
}
|
|
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3"))
|
|
s3_pb.RegisterSeaweedS3IamCacheServer(grpcS, s3ApiServer)
|
|
s3_lifecycle_pb.RegisterSeaweedS3LifecycleInternalServer(grpcS, s3ApiServer)
|
|
reflection.Register(grpcS)
|
|
if grpcLocalL != nil {
|
|
go grpcS.Serve(grpcLocalL)
|
|
}
|
|
go grpcS.Serve(grpcL)
|
|
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
|
|
stopGrpcServer := func() { gracefulStopGrpc(grpcS, 15*time.Second) }
|
|
|
|
if *s3opt.tlsPrivateKey != "" {
|
|
// Check for port conflict when both HTTP and HTTPS are enabled on the same port
|
|
if *s3opt.portHttps > 0 && *s3opt.portHttps == *s3opt.port {
|
|
glog.Fatalf("S3 API Server error: -s3.port.https (%d) cannot be the same as -s3.port (%d)", *s3opt.portHttps, *s3opt.port)
|
|
}
|
|
|
|
getCert, certProvider, err := security.NewReloadingServerCertificate(*s3opt.tlsCertificate, *s3opt.tlsPrivateKey)
|
|
if err != nil {
|
|
glog.Fatalf("S3 API Server failed to load HTTPS certificate: %v", err)
|
|
}
|
|
grace.OnInterrupt(certProvider.Close)
|
|
|
|
caCertPool, err := loadClientCAPool(*s3opt.tlsCACertificate)
|
|
if err != nil {
|
|
glog.Fatalf("%v", err)
|
|
}
|
|
|
|
clientAuth := tls.NoClientCert
|
|
if *s3opt.tlsVerifyClientCert {
|
|
clientAuth = tls.RequireAndVerifyClientCert
|
|
}
|
|
|
|
tlsConfig := &tls.Config{
|
|
GetCertificate: getCert,
|
|
ClientAuth: clientAuth,
|
|
ClientCAs: caCertPool,
|
|
}
|
|
err = security.FixTlsConfig(util.GetViper(), tlsConfig)
|
|
if err != nil {
|
|
glog.Fatalf("error with tls config: %v", err)
|
|
}
|
|
if *s3opt.portHttps == 0 {
|
|
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.port)
|
|
if s3ApiLocalListener != nil {
|
|
localServer := newHttpServer(vectorsHandler, tlsConfig)
|
|
httpServers = append(httpServers, localServer)
|
|
go func() {
|
|
if err := localServer.ServeTLS(s3ApiLocalListener, "", ""); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
httpS := newHttpServer(vectorsHandler, tlsConfig)
|
|
httpServers = append(httpServers, httpS)
|
|
shutdown := s3opt.newShutdown(stopGrpcServer, s3ApiServer, httpServers)
|
|
if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
// Serve returns when listeners close, before active requests finish.
|
|
shutdown()
|
|
} else {
|
|
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", version.Version(), *s3opt.portHttps)
|
|
s3ApiListenerHttps, s3ApiLocalListenerHttps, err := util.NewIpAndLocalListeners(
|
|
*s3opt.bindIp, *s3opt.portHttps, idleTimeoutSeconds(s3opt.idleTimeout), listenOnLocalhost)
|
|
if err != nil {
|
|
glog.Fatalf("S3 API HTTPS listener on %s:%d error: %v", *s3opt.bindIp, *s3opt.portHttps, err)
|
|
}
|
|
if s3ApiLocalListenerHttps != nil {
|
|
localHttpsServer := newHttpServer(vectorsHandler, tlsConfig)
|
|
httpServers = append(httpServers, localHttpsServer)
|
|
go func() {
|
|
if err := localHttpsServer.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
httpsServer := newHttpServer(vectorsHandler, tlsConfig)
|
|
httpServers = append(httpServers, httpsServer)
|
|
go func() {
|
|
if err := httpsServer.ServeTLS(s3ApiListenerHttps, "", ""); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
|
|
glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", version.Version(), *s3opt.port)
|
|
if s3ApiLocalListener != nil {
|
|
localServer := newHttpServer(vectorsHandler, nil)
|
|
httpServers = append(httpServers, localServer)
|
|
go func() {
|
|
if err := localServer.Serve(s3ApiLocalListener); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
httpS := newHttpServer(vectorsHandler, nil)
|
|
httpServers = append(httpServers, httpS)
|
|
shutdown := s3opt.newShutdown(stopGrpcServer, s3ApiServer, httpServers)
|
|
if err = httpS.Serve(s3ApiListener); err != nil && err != http.ErrServerClosed {
|
|
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
|
}
|
|
// Serve returns when listeners close, before active requests finish.
|
|
shutdown()
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiServer, httpServers []*http.Server) func() {
|
|
shutdown := newGracefulShutdown(stopGrpc, s3ApiServer.Shutdown, httpServers...)
|
|
var cancelled <-chan struct{}
|
|
if s3opt.shutdownCtx != nil {
|
|
cancelled = s3opt.shutdownCtx.Done()
|
|
go func() {
|
|
<-cancelled
|
|
shutdown()
|
|
}()
|
|
}
|
|
preStop := sync.OnceFunc(preStopWait(s3ApiServer.StartDraining, s3opt.preStopDelay(), cancelled))
|
|
// The wait has to finish before weed mini cancels shutdownCtx, and before
|
|
// weed server shuts down the filer and volume server, whose hooks may be
|
|
// registered after this one.
|
|
beforeMiniClientsShutdown(preStop)
|
|
grace.OnInterruptFirst(preStop)
|
|
grace.OnInterrupt(shutdown)
|
|
return shutdown
|
|
}
|
|
|
|
func (s3opt *S3Options) preStopDelay() time.Duration {
|
|
if s3opt.preStopSeconds == nil || *s3opt.preStopSeconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(*s3opt.preStopSeconds) * time.Second
|
|
}
|
|
|
|
func preStopWait(startDraining func(), delay time.Duration, cancelled <-chan struct{}) func() {
|
|
return func() {
|
|
if delay <= 0 {
|
|
return
|
|
}
|
|
startDraining()
|
|
glog.V(0).Infof("S3 API Server failing health checks for %s before shutdown", delay)
|
|
timer := time.NewTimer(delay)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
case <-cancelled:
|
|
}
|
|
}
|
|
}
|
|
|
|
// vendsCredentials reports whether -iceberg.credentialRole is set, which
|
|
// both catalogs vend through, whichever of them runs.
|
|
func (s3opt *S3Options) vendsCredentials() bool {
|
|
return s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != ""
|
|
}
|
|
|
|
// setCredentialRole hands -iceberg.credentialRole to the S3 server.
|
|
func (s3opt *S3Options) setCredentialRole(s3ApiServer *s3api.S3ApiServer) {
|
|
if !s3opt.vendsCredentials() {
|
|
return
|
|
}
|
|
duration := int64(0)
|
|
if s3opt.icebergCredentialDuration != nil {
|
|
duration = int64(*s3opt.icebergCredentialDuration)
|
|
}
|
|
s3ApiServer.SetIcebergCredentialRole(*s3opt.icebergCredentialRole, duration)
|
|
}
|
|
|
|
// startIcebergServer starts the Iceberg REST Catalog server on a separate port.
|
|
func (s3opt *S3Options) startIcebergServer(s3ApiServer *s3api.S3ApiServer, listenOnLocalhost bool) *http.Server {
|
|
icebergRouter := mux.NewRouter().SkipClean(true)
|
|
// warehouse/parent query values may legally contain ';', which Go's
|
|
// url.ParseQuery would otherwise drop
|
|
icebergRouter.Use(util_http.EscapeSemicolonsInQuery)
|
|
|
|
// Create Iceberg server using the S3ApiServer as filer client
|
|
icebergServer := iceberg.NewServer(s3ApiServer, s3ApiServer)
|
|
icebergServer.SetCredentialValidator(s3ApiServer)
|
|
if s3opt.vendsCredentials() {
|
|
icebergServer.SetCredentialVendor(icebergCredentialVendor{s3ApiServer})
|
|
}
|
|
icebergServer.SetS3Endpoint(s3opt.deriveS3AdvertisedEndpoint())
|
|
icebergServer.RegisterRoutes(icebergRouter)
|
|
|
|
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portIceberg)
|
|
icebergListener, icebergLocalListener, err := util.NewIpAndLocalListeners(
|
|
*s3opt.bindIp, *s3opt.portIceberg, idleTimeoutSeconds(s3opt.idleTimeout), listenOnLocalhost)
|
|
if err != nil {
|
|
// The catalog is auxiliary to the S3 gateway; a port conflict must not
|
|
// take the whole server down.
|
|
glog.Errorf("Iceberg REST Catalog listener on %s error: %v", listenAddress, err)
|
|
return nil
|
|
}
|
|
s3opt.icebergBound.Store(true)
|
|
|
|
glog.V(0).Infof("Start Iceberg REST Catalog Server at http://%s", listenAddress)
|
|
|
|
httpS := newHttpServer(icebergRouter, nil)
|
|
// Serve on localhost as well if we're bound to a different interface
|
|
if icebergLocalListener != nil {
|
|
go func() {
|
|
if err := httpS.Serve(icebergLocalListener); err != nil && err != http.ErrServerClosed {
|
|
glog.V(0).Infof("Iceberg localhost listener error: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
if err := httpS.Serve(icebergListener); err != nil && err != http.ErrServerClosed {
|
|
glog.Errorf("Iceberg REST Catalog Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
return httpS
|
|
}
|
|
|
|
// newLanceServer builds the Lance Namespace catalog. It shares the Iceberg
|
|
// catalog's credential role: one deployment vends table credentials one way,
|
|
// whichever catalog the client speaks to.
|
|
func (s3opt *S3Options) newLanceServer(s3ApiServer *s3api.S3ApiServer) *lance.Server {
|
|
lanceServer := lance.NewServer(s3ApiServer, s3ApiServer)
|
|
lanceServer.SetCredentialValidator(s3ApiServer)
|
|
lanceServer.SetSigningKey(s3ApiServer.FilerSigningKey)
|
|
if s3opt.vendsCredentials() {
|
|
lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer})
|
|
}
|
|
lanceServer.SetS3Endpoint(s3opt.deriveLanceStorageEndpoint())
|
|
lanceServer.SetS3Region(s3tables.DefaultRegion)
|
|
return lanceServer
|
|
}
|
|
|
|
// startLanceServer serves the Lance Namespace catalog on its own port.
|
|
func (s3opt *S3Options) startLanceServer(listenOnLocalhost bool) *http.Server {
|
|
lanceRouter := mux.NewRouter().SkipClean(true)
|
|
lanceRouter.Use(util_http.EscapeSemicolonsInQuery)
|
|
s3opt.lanceServer.RegisterRoutes(lanceRouter)
|
|
|
|
listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.portLance)
|
|
lanceListener, lanceLocalListener, err := util.NewIpAndLocalListeners(
|
|
*s3opt.bindIp, *s3opt.portLance, idleTimeoutSeconds(s3opt.idleTimeout), listenOnLocalhost)
|
|
if err != nil {
|
|
glog.Errorf("Lance Namespace listener on %s error: %v", listenAddress, err)
|
|
return nil
|
|
}
|
|
s3opt.lanceBound.Store(true)
|
|
|
|
glog.V(0).Infof("Start Lance Namespace Server at http://%s", listenAddress)
|
|
|
|
httpS := newHttpServer(lanceRouter, nil)
|
|
if lanceLocalListener != nil {
|
|
go func() {
|
|
if err := httpS.Serve(lanceLocalListener); err != nil && err != http.ErrServerClosed {
|
|
glog.V(0).Infof("Lance localhost listener error: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
go func() {
|
|
if err := httpS.Serve(lanceListener); err != nil && err != http.ErrServerClosed {
|
|
glog.Errorf("Lance Namespace Server Fail to serve: %v", err)
|
|
}
|
|
}()
|
|
return httpS
|
|
}
|
|
|
|
// vectorsRouter builds the S3 Vectors API router — weed-vector servers
|
|
// register to the master and the backend resolves them lazily, so a server
|
|
// that comes up after the gateway still gets picked up on first query.
|
|
func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOption grpc.DialOption, masterAddresses []pb.ServerAddress, filerGroup string) *mux.Router {
|
|
if len(masterAddresses) == 0 {
|
|
glog.Warningf("S3 Vectors API disabled: no weed-vector registered on master")
|
|
return nil
|
|
}
|
|
backend := s3vectors.NewVectorDiscovery(func() []pb.ServerAddress { return masterAddresses }, grpcDialOption, filerGroup)
|
|
// In-process describe — the router resolves locations and vended
|
|
// storage options through the catalog directly, never over HTTP. The
|
|
// lookup itself runs under the trusted identity: the caller's vector
|
|
// permission already authorized the query, and catalog grants must not
|
|
// gate an internal resolution.
|
|
describe := func(ctx context.Context, id []string) (string, map[string]string, error) {
|
|
if s3opt.lanceServer == nil {
|
|
return "", nil, fmt.Errorf("lance catalog is not running")
|
|
}
|
|
r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
|
|
return s3opt.lanceServer.DescribeTableInternal(r, id)
|
|
}
|
|
vectorsRouter := mux.NewRouter().SkipClean(true)
|
|
|
|
vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend)
|
|
vectorsServer.SetRegion(s3tables.DefaultRegion)
|
|
vectorsServer.SetAccountID(s3tables.DefaultAccountID)
|
|
vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured)
|
|
vectorsServer.SetDescribe(describe)
|
|
vectorsServer.SetLimiter(s3ApiServer.Limit)
|
|
vectorsServer.SetRequestLimits(int64(*s3opt.concurrentFileUploadLimit), int64(*s3opt.concurrentUploadLimitMB)*1024*1024)
|
|
vectorsServer.RegisterRoutes(vectorsRouter)
|
|
return vectorsRouter
|
|
}
|
|
|
|
// s3Handler serves router, and vectors for Vectors API requests when set. mux
|
|
// matches Queries() before router middleware runs, so the escape wraps the
|
|
// router for routing and authorization to read one query.
|
|
func s3Handler(router, vectors *mux.Router) http.Handler {
|
|
main := util_http.EscapeSemicolonsInQuery(router)
|
|
if vectors == nil {
|
|
return main
|
|
}
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if isVectorsRequest(r) {
|
|
vectors.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
main.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// isVectorsRequest routes by the SigV4 credential scope — the s3vectors
|
|
// service name lives there, not in the path.
|
|
func isVectorsRequest(r *http.Request) bool {
|
|
if v := r.Header.Get("Authorization"); v != "" {
|
|
return strings.Contains(v, "/s3vectors/")
|
|
}
|
|
return strings.Contains(r.URL.Query().Get("X-Amz-Credential"), "/s3vectors/")
|
|
}
|
|
|
|
func loadClientCAPool(caCertFile string) (*x509.CertPool, error) {
|
|
pool := x509.NewCertPool()
|
|
if caCertFile == "" {
|
|
return pool, nil
|
|
}
|
|
pem, err := os.ReadFile(caCertFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error reading CA certificate: %w", err)
|
|
}
|
|
if !pool.AppendCertsFromPEM(pem) {
|
|
return nil, fmt.Errorf("no valid certificates found in %s", caCertFile)
|
|
}
|
|
return pool, nil
|
|
}
|
|
|
|
// deriveLanceStorageEndpoint picks the endpoint the Lance namespace puts in
|
|
// storage_options. It falls back to the advertised -ip where the Iceberg
|
|
// derivation gives up, because the two clients are not in the same position: a
|
|
// Spark or Trino Iceberg client brings its own s3.endpoint and advertising the
|
|
// wrong one hijacks it, whereas storage_options is the only place a Lance
|
|
// client learns where the store is. Without one, object_store quietly falls
|
|
// back to real AWS S3 and the failure reads like a credentials problem.
|
|
func (s3opt *S3Options) deriveLanceStorageEndpoint() string {
|
|
if endpoint := s3opt.deriveS3AdvertisedEndpoint(); endpoint != "" {
|
|
return endpoint
|
|
}
|
|
host := ""
|
|
if s3opt.ip != nil {
|
|
host = *s3opt.ip
|
|
}
|
|
switch host {
|
|
case "", "0.0.0.0", "::", "[::]":
|
|
return ""
|
|
}
|
|
scheme := "http"
|
|
port := 0
|
|
if s3opt.port != nil {
|
|
port = *s3opt.port
|
|
}
|
|
if s3opt.tlsPrivateKey != nil && *s3opt.tlsPrivateKey != "" {
|
|
scheme = "https"
|
|
if s3opt.portHttps != nil && *s3opt.portHttps > 0 {
|
|
port = *s3opt.portHttps
|
|
}
|
|
}
|
|
return fmt.Sprintf("%s://%s", scheme, util.JoinHostPort(host, port))
|
|
}
|
|
|
|
// deriveS3AdvertisedEndpoint builds the S3 endpoint URL to advertise to
|
|
// Iceberg catalog clients as part of LoadTable FileIO config. To avoid
|
|
// hijacking correctly-configured clients (Spark/Trino/PyIceberg all bring
|
|
// their own s3.endpoint), advertising is strictly opt-in and returns ""
|
|
// whenever no reliable value is available:
|
|
// - -s3.externalUrl / S3_EXTERNAL_URL wins and supports reverse-proxy
|
|
// deployments.
|
|
// - Otherwise the bind IP is used only when it is explicit and not a
|
|
// wildcard (0.0.0.0 / ::), with the scheme picked from TLS config and
|
|
// IPv6 literals bracketed via util.JoinHostPort.
|
|
//
|
|
// See issue #9103.
|
|
func (s3opt *S3Options) deriveS3AdvertisedEndpoint() string {
|
|
if ext := strings.TrimRight(s3opt.resolveExternalUrl(), "/"); ext != "" {
|
|
return ext
|
|
}
|
|
|
|
host := ""
|
|
if s3opt.bindIp != nil {
|
|
host = *s3opt.bindIp
|
|
}
|
|
switch host {
|
|
case "", "0.0.0.0", "::", "[::]":
|
|
return ""
|
|
}
|
|
|
|
scheme := "http"
|
|
port := 0
|
|
if s3opt.port != nil {
|
|
port = *s3opt.port
|
|
}
|
|
if s3opt.tlsPrivateKey != nil && *s3opt.tlsPrivateKey != "" {
|
|
scheme = "https"
|
|
if s3opt.portHttps != nil && *s3opt.portHttps > 0 {
|
|
port = *s3opt.portHttps
|
|
}
|
|
}
|
|
return fmt.Sprintf("%s://%s", scheme, util.JoinHostPort(host, port))
|
|
}
|
|
|
|
// icebergCredentialVendor adapts the S3 gateway's STS-backed vending to the
|
|
// catalog's interface, keeping the two packages independent of each other.
|
|
type icebergCredentialVendor struct {
|
|
server *s3api.S3ApiServer
|
|
}
|
|
|
|
func (v icebergCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*iceberg.VendedCredentials, error) {
|
|
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
|
|
if err != nil || credentials == nil {
|
|
return nil, err
|
|
}
|
|
return &iceberg.VendedCredentials{
|
|
AccessKeyID: credentials.AccessKeyID,
|
|
SecretAccessKey: credentials.SecretAccessKey,
|
|
SessionToken: credentials.SessionToken,
|
|
Expiration: credentials.Expiration,
|
|
}, nil
|
|
}
|
|
|
|
// lanceCredentialVendor adapts the S3 gateway's STS-backed vending to the Lance
|
|
// namespace's interface, keeping the two packages independent of each other.
|
|
type lanceCredentialVendor struct {
|
|
server *s3api.S3ApiServer
|
|
}
|
|
|
|
func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*lance.VendedCredentials, error) {
|
|
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
|
|
if err != nil || credentials == nil {
|
|
return nil, err
|
|
}
|
|
return &lance.VendedCredentials{
|
|
AccessKeyID: credentials.AccessKeyID,
|
|
SecretAccessKey: credentials.SecretAccessKey,
|
|
SessionToken: credentials.SessionToken,
|
|
Expiration: credentials.Expiration,
|
|
}, nil
|
|
}
|