package client import ( "crypto/tls" "crypto/x509" "fmt" "io" "net/http" "net/url" "os" "strings" "sync" "time" "google.golang.org/grpc/credentials/tls/certprovider" "github.com/seaweedfs/seaweedfs/weed/security/certreload" util "github.com/seaweedfs/seaweedfs/weed/util" "github.com/spf13/viper" ) var ( loadSecurityConfigOnce sync.Once ) // Intra-cluster peers (volume/filer/master) answer headers within milliseconds. // A peer that is TCP-reachable but not answering -- a volume server still loading // after a restart, or a stale keep-alive to a container that returned on a new IP // -- otherwise blocks a chunk read or a replicated write forever. The response // timeout bounds that wait so the read fails over to another replica and the // write fails fast to retry; the idle timeout evicts sockets to a departed server. const ( responseHeaderTimeout = 30 * time.Second idleConnTimeout = 90 * time.Second ) type HTTPClient struct { Client *http.Client Transport *http.Transport expectHttpsScheme bool // certProvider, when non-nil, owns a background refresh goroutine for // the client mTLS cert/key pair. Close() must be called to stop it. certProvider certprovider.Provider } // Close stops any background cert refresh goroutine. Safe to call on a // client that was constructed without mTLS. Existing pooled connections // are also closed via CloseIdleConnections. func (httpClient *HTTPClient) Close() { if httpClient == nil { return } if httpClient.certProvider != nil { httpClient.certProvider.Close() httpClient.certProvider = nil } if httpClient.Client != nil { httpClient.Client.CloseIdleConnections() } } func (httpClient *HTTPClient) Do(req *http.Request) (*http.Response, error) { req.URL.Scheme = httpClient.GetHttpScheme() return httpClient.Client.Do(req) } func (httpClient *HTTPClient) Get(url string) (resp *http.Response, err error) { url, err = httpClient.NormalizeHttpScheme(url) if err != nil { return nil, err } return httpClient.Client.Get(url) } func (httpClient *HTTPClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) { url, err = httpClient.NormalizeHttpScheme(url) if err != nil { return nil, err } return httpClient.Client.Post(url, contentType, body) } func (httpClient *HTTPClient) PostForm(url string, data url.Values) (resp *http.Response, err error) { url, err = httpClient.NormalizeHttpScheme(url) if err != nil { return nil, err } return httpClient.Client.PostForm(url, data) } func (httpClient *HTTPClient) Head(url string) (resp *http.Response, err error) { url, err = httpClient.NormalizeHttpScheme(url) if err != nil { return nil, err } return httpClient.Client.Head(url) } func (httpClient *HTTPClient) CloseIdleConnections() { httpClient.Client.CloseIdleConnections() } func (httpClient *HTTPClient) GetClientTransport() *http.Transport { return httpClient.Transport } func (httpClient *HTTPClient) GetHttpScheme() string { if httpClient.expectHttpsScheme { return "https" } return "http" } func (httpClient *HTTPClient) NormalizeHttpScheme(rawURL string) (string, error) { expectedScheme := httpClient.GetHttpScheme() if !(strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")) { return expectedScheme + "://" + rawURL, nil } parsedURL, err := url.Parse(rawURL) if err != nil { return "", err } if expectedScheme != parsedURL.Scheme { parsedURL.Scheme = expectedScheme } return parsedURL.String(), nil } func NewHttpClient(clientName ClientName, opts ...HttpClientOpt) (*HTTPClient, error) { httpClient := HTTPClient{} httpClient.expectHttpsScheme = checkIsHttpsClientEnabled(clientName) var tlsConfig *tls.Config = nil if httpClient.expectHttpsScheme { certFileName, keyFileName, hasClientCert, err := clientCertPaths(clientName) if err != nil { return nil, err } clientCaCert, clientCaCertName, err := getClientCaCert(clientName) if err != nil { return nil, err } if hasClientCert || len(clientCaCert) != 0 { caCertPool, err := createHTTPClientCertPool(clientCaCert, clientCaCertName) if err != nil { return nil, err } tlsConfig = &tls.Config{ RootCAs: caCertPool, InsecureSkipVerify: false, } if hasClientCert { getClientCert, provider, err := certreload.NewClientGetCertificate(certFileName, keyFileName) if err != nil { return nil, fmt.Errorf("error loading client certificate and key: %s", err) } tlsConfig.GetClientCertificate = getClientCert httpClient.certProvider = provider } } if getBoolOptionFromSecurityConfiguration(clientName, "insecure_skip_verify") { if tlsConfig == nil { tlsConfig = &tls.Config{} } tlsConfig.InsecureSkipVerify = true } } httpClient.Transport = &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, TLSClientConfig: tlsConfig, // Bind outbound HTTP to the -ip.bind source address. DialContext: util.OutboundDialContext, ResponseHeaderTimeout: responseHeaderTimeout, IdleConnTimeout: idleConnTimeout, } httpClient.Client = &http.Client{ Transport: httpClient.Transport, } for _, opt := range opts { opt(&httpClient) } return &httpClient, nil } func getStringOptionFromSecurityConfiguration(clientName ClientName, stringOptionName string) string { util.LoadSecurityConfiguration() return viper.GetString(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), stringOptionName)) } func getBoolOptionFromSecurityConfiguration(clientName ClientName, boolOptionName string) bool { util.LoadSecurityConfiguration() return viper.GetBool(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), boolOptionName)) } func checkIsHttpsClientEnabled(clientName ClientName) bool { return getBoolOptionFromSecurityConfiguration(clientName, "enabled") } func getFileContentFromSecurityConfiguration(clientName ClientName, fileType string) ([]byte, string, error) { if fileName := getStringOptionFromSecurityConfiguration(clientName, fileType); fileName != "" { fileContent, err := os.ReadFile(fileName) if err != nil { return nil, fileName, err } return fileContent, fileName, err } return nil, "", nil } // clientCertPaths reads the https..{cert,key} paths from the // security config, validates they're either both set or both empty, and // returns them along with a hasClientCert flag. Loading is deferred to // certreload so the cert/key pair is picked up from disk on rotation. func clientCertPaths(clientName ClientName) (certFile, keyFile string, hasClientCert bool, err error) { certFile = getStringOptionFromSecurityConfiguration(clientName, "cert") keyFile = getStringOptionFromSecurityConfiguration(clientName, "key") if certFile == "" && keyFile == "" { return "", "", false, nil } if certFile == "" || keyFile == "" { return "", "", false, fmt.Errorf("https.%s: both cert and key must be set (got cert=%q key=%q)", clientName.LowerCaseString(), certFile, keyFile) } return certFile, keyFile, true, nil } func getClientCaCert(clientName ClientName) ([]byte, string, error) { return getFileContentFromSecurityConfiguration(clientName, "ca") } // NewHttpClientWithTLS creates an HTTPClient with explicit TLS certificate // parameters instead of reading from the global security configuration. // This is used by filer.sync to create per-cluster HTTP clients when clusters // use different certificates. func NewHttpClientWithTLS(certFile, keyFile, caFile string, insecureSkipVerify bool, opts ...HttpClientOpt) (*HTTPClient, error) { httpClient := HTTPClient{} httpClient.expectHttpsScheme = true var tlsConfig *tls.Config if (certFile == "") != (keyFile == "") { return nil, fmt.Errorf("both cert and key are required for mTLS, got cert=%q key=%q", certFile, keyFile) } var getClientCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error) if certFile != "" && keyFile != "" { cb, provider, err := certreload.NewClientGetCertificate(certFile, keyFile) if err != nil { return nil, fmt.Errorf("error loading client certificate and key: %s", err) } getClientCert = cb httpClient.certProvider = provider } // closeProviderOnError ensures the cert reloader's background refresh // goroutine is shut down if any subsequent step fails before we hand // the client back to the caller. closeProviderOnError := func() { if httpClient.certProvider != nil { httpClient.certProvider.Close() httpClient.certProvider = nil } } var caCertPool *x509.CertPool if caFile != "" { caCert, err := os.ReadFile(caFile) if err != nil { closeProviderOnError() return nil, fmt.Errorf("error reading CA cert %s: %s", caFile, err) } caCertPool, err = createHTTPClientCertPool(caCert, caFile) if err != nil { closeProviderOnError() return nil, err } } if getClientCert != nil || caCertPool != nil || insecureSkipVerify { tlsConfig = &tls.Config{ GetClientCertificate: getClientCert, RootCAs: caCertPool, InsecureSkipVerify: insecureSkipVerify, } } httpClient.Transport = &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, TLSClientConfig: tlsConfig, // Bind outbound HTTP to the -ip.bind source address. DialContext: util.OutboundDialContext, ResponseHeaderTimeout: responseHeaderTimeout, IdleConnTimeout: idleConnTimeout, } httpClient.Client = &http.Client{ Transport: httpClient.Transport, } for _, opt := range opts { opt(&httpClient) } return &httpClient, nil } func createHTTPClientCertPool(certContent []byte, fileName string) (*x509.CertPool, error) { certPool := x509.NewCertPool() if len(certContent) == 0 { return certPool, nil } ok := certPool.AppendCertsFromPEM(certContent) if !ok { return nil, fmt.Errorf("error processing certificate in %s", fileName) } return certPool, nil }