seaweedfs/weed/server/common_test.go

package weed_server

import (
	"bytes"
	"context"
	"errors"
	"io"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb"
)

func TestParseURL(t *testing.T) {
	if vid, fid, _, _, _ := parseURLPath("/1,06dfa8a684"); true {
		if vid != "1" {
			t.Errorf("fail to parse vid: %s", vid)
		}
		if fid != "06dfa8a684" {
			t.Errorf("fail to parse fid: %s", fid)
		}
	}
	if vid, fid, _, _, _ := parseURLPath("/1,06dfa8a684_1"); true {
		if vid != "1" {
			t.Errorf("fail to parse vid: %s", vid)
		}
		if fid != "06dfa8a684_1" {
			t.Errorf("fail to parse fid: %s", fid)
		}
		if sepIndex := strings.LastIndex(fid, "_"); sepIndex > 0 {
			fid = fid[:sepIndex]
		}
		if fid != "06dfa8a684" {
			t.Errorf("fail to parse fid: %s", fid)
		}
	}
}

func TestWriteJsonNoJSONP(t *testing.T) {
	// callback= must be ignored; response is always application/json with nosniff.
	cases := []string{"", "myCb", "<script>alert(1)</script>"}
	for _, cb := range cases {
		t.Run("callback="+cb, func(t *testing.T) {
			url := "/x"
			if cb != "" {
				url += "?callback=" + cb
			}
			r := httptest.NewRequest(http.MethodGet, url, nil)
			w := httptest.NewRecorder()
			if err := writeJson(w, r, http.StatusOK, map[string]string{"k": "v"}); err != nil {
				t.Fatalf("writeJson: %v", err)
			}
			if w.Code != http.StatusOK {
				t.Errorf("status: got %d want 200", w.Code)
			}
			if got := w.Header().Get("Content-Type"); got != "application/json" {
				t.Errorf("Content-Type: got %q want application/json", got)
			}
			if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" {
				t.Errorf("X-Content-Type-Options: got %q want nosniff", got)
			}
			if got := w.Body.String(); got != `{"k":"v"}` {
				t.Errorf("body: got %q want %q", got, `{"k":"v"}`)
			}
		})
	}
}

// A POST to a path with a doubled slash must reach the handler at the cleaned,
// still-decoded path. Without CleanPathHandler, http.ServeMux answers with a
// redirect whose Location percent-encodes the already-escaped path a second
// time (golang/go#79897), and a client following it creates directories
// literally named "%E8%B4%9F..." (seaweedfs#11125).
//
// RequestURI must follow the cleaned path as well: PostHandler derives storage
// rules, bucket and read-only checks from it, so it has to agree with URL.Path.
// wantRequestURI defaults to wantEscaped.
func TestCleanPathHandlerServesCleanedPathWithoutRedirect(t *testing.T) {
	tests := []struct {
		name           string
		target         string
		wantPath       string
		wantEscaped    string
		wantRequestURI string
	}{
		{
			name:        "double slash before non-ascii segments",
			target:      "/Image/2026-09-04//负极全景/OK渲染图/test123_残片正光_拼图_105525641.jpg",
			wantPath:    "/Image/2026-09-04/负极全景/OK渲染图/test123_残片正光_拼图_105525641.jpg",
			wantEscaped: "/Image/2026-09-04/%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/OK%E6%B8%B2%E6%9F%93%E5%9B%BE/test123_%E6%AE%8B%E7%89%87%E6%AD%A3%E5%85%89_%E6%8B%BC%E5%9B%BE_105525641.jpg",
		},
		{
			name:        "already percent-encoded by the client",
			target:      "/Image//%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/a.jpg",
			wantPath:    "/Image/负极全景/a.jpg",
			wantEscaped: "/Image/%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/a.jpg",
		},
		{
			name:        "trailing slash marks a directory and is kept",
			target:      "/Image//负极全景/",
			wantPath:    "/Image/负极全景/",
			wantEscaped: "/Image/%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/",
		},
		{
			name:        "dot segments are resolved",
			target:      "/Image/./tmp/../负极全景/a.jpg",
			wantPath:    "/Image/负极全景/a.jpg",
			wantEscaped: "/Image/%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/a.jpg",
		},
		{
			name:           "dot segments crossing a bucket move RequestURI to the written bucket, query kept",
			target:         "/buckets/a/../b/f.jpg?collection=c&ttl=1d",
			wantPath:       "/buckets/b/f.jpg",
			wantEscaped:    "/buckets/b/f.jpg",
			wantRequestURI: "/buckets/b/f.jpg?collection=c&ttl=1d",
		},
		{
			name:           "canonical path is passed through untouched",
			target:         "/Image/负极全景/a.jpg",
			wantPath:       "/Image/负极全景/a.jpg",
			wantEscaped:    "/Image/%E8%B4%9F%E6%9E%81%E5%85%A8%E6%99%AF/a.jpg",
			wantRequestURI: "/Image/负极全景/a.jpg",
		},
		{
			name:        "encoded slash is not a separator and is preserved",
			target:      "/Image//a%2F%2Fb/c.jpg",
			wantPath:    "/Image/a//b/c.jpg",
			wantEscaped: "/Image/a%2F%2Fb/c.jpg",
		},
		{
			name:        "root",
			target:      "//",
			wantPath:    "/",
			wantEscaped: "/",
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			var gotPath, gotEscaped, gotRequestURI string
			mux := http.NewServeMux()
			mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
				gotPath, gotEscaped, gotRequestURI = r.URL.Path, r.URL.EscapedPath(), r.RequestURI
			})

			w := httptest.NewRecorder()
			CleanPathHandler(mux).ServeHTTP(w, httptest.NewRequest(http.MethodPost, tc.target, strings.NewReader("data")))

			if w.Code != http.StatusOK {
				t.Fatalf("status: got %d (Location %q), want 200 with no redirect", w.Code, w.Header().Get("Location"))
			}
			if gotPath != tc.wantPath {
				t.Errorf("r.URL.Path: got %q want %q", gotPath, tc.wantPath)
			}
			if gotEscaped != tc.wantEscaped {
				t.Errorf("r.URL.EscapedPath(): got %q want %q", gotEscaped, tc.wantEscaped)
			}
			wantRequestURI := tc.wantRequestURI
			if wantRequestURI == "" {
				wantRequestURI = tc.wantEscaped
			}
			if gotRequestURI != wantRequestURI {
				t.Errorf("r.RequestURI: got %q want %q", gotRequestURI, wantRequestURI)
			}
		})
	}
}

func TestWriteJsonPrettyDoesNotReadMultipartBody(t *testing.T) {
	var form bytes.Buffer
	mw := multipart.NewWriter(&form)
	if err := mw.WriteField("pretty", "1"); err != nil {
		t.Fatalf("write pretty field: %v", err)
	}
	part, err := mw.CreateFormFile("file", "test.txt")
	if err != nil {
		t.Fatalf("create form file: %v", err)
	}
	if _, err := part.Write([]byte("hello")); err != nil {
		t.Fatalf("write form file: %v", err)
	}
	if err := mw.Close(); err != nil {
		t.Fatalf("close multipart writer: %v", err)
	}

	body := &countingReadCloser{Reader: bytes.NewReader(form.Bytes())}
	r := httptest.NewRequest(http.MethodPost, "/x", body)
	r.Header.Set("Content-Type", mw.FormDataContentType())
	w := httptest.NewRecorder()

	if err := writeJson(w, r, http.StatusTooManyRequests, map[string]string{"error": "busy"}); err != nil {
		t.Fatalf("writeJson: %v", err)
	}

	if body.reads != 0 {
		t.Fatalf("writeJson read multipart body %d times", body.reads)
	}
	if got, want := w.Body.String(), `{"error":"busy"}`; got != want {
		t.Fatalf("body: got %q want %q", got, want)
	}
}

type countingReadCloser struct {
	io.Reader
	reads int
}

func (c *countingReadCloser) Read(p []byte) (int, error) {
	c.reads++
	return c.Reader.Read(p)
}

func (c *countingReadCloser) Close() error {
	return nil
}

// A write error after the body has flushed must not complete as 200 with the
// full Content-Length: the unflushed tail is discarded and the connection
// aborted mid-body.
func TestProcessRangeRequestWriteErrorAfterBodyStarted(t *testing.T) {
	payload := bytes.Repeat([]byte("x"), 200*1024)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_ = ProcessRangeRequest(r, w, int64(len(payload)), "application/octet-stream", func(offset int64, size int64) (filer.DoStreamContent, error) {
			return func(writer io.Writer) error {
				// Page-sized writes, as readNeedleDataInto does. The tail sits
				// in the 128KiB buffer; flushing it would finish the 200.
				for off := 0; off < len(payload); off += 4096 {
					end := off + 4096
					if end > len(payload) {
						end = len(payload)
					}
					if _, werr := writer.Write(payload[off:end]); werr != nil {
						return werr
					}
				}
				return errors.New("ReadNeedleData checksum mismatch")
			}, nil
		})
	}))
	defer srv.Close()

	resp, err := srv.Client().Get(srv.URL)
	if err != nil {
		t.Fatalf("request: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("status %d, want the committed 200", resp.StatusCode)
	}
	body, readErr := io.ReadAll(resp.Body)
	if readErr == nil {
		t.Fatalf("body completed with %d bytes; the erroring tail must abort it", len(body))
	}
}

func TestProcessRangeRequestWriteErrorBeforeBody(t *testing.T) {
	r := httptest.NewRequest(http.MethodGet, "/test.bin", nil)
	w := httptest.NewRecorder()
	err := ProcessRangeRequest(r, w, 100, "application/octet-stream", func(offset int64, size int64) (filer.DoStreamContent, error) {
		return func(writer io.Writer) error {
			return errors.New("ReadNeedleData checksum mismatch")
		}, nil
	})
	if err == nil {
		t.Fatal("expected write error")
	}
	if w.Code != http.StatusInternalServerError {
		t.Fatalf("status %d, want 500, body %q", w.Code, w.Body.String())
	}
	if !strings.Contains(w.Body.String(), "checksum") {
		t.Fatalf("body %q, want the checksum error", w.Body.String())
	}
}

func TestProcessRangeRequestLargeBody(t *testing.T) {
	payload := bytes.Repeat([]byte("y"), 200*1024)
	r := httptest.NewRequest(http.MethodGet, "/test.bin", nil)
	w := httptest.NewRecorder()
	err := ProcessRangeRequest(r, w, int64(len(payload)), "application/octet-stream", func(offset int64, size int64) (filer.DoStreamContent, error) {
		return func(writer io.Writer) error {
			_, werr := writer.Write(payload)
			return werr
		}, nil
	})
	if err != nil {
		t.Fatalf("write: %v", err)
	}
	if w.Code != http.StatusOK {
		t.Fatalf("status %d, want 200", w.Code)
	}
	if !bytes.Equal(w.Body.Bytes(), payload) {
		t.Fatalf("body len %d, want %d", w.Body.Len(), len(payload))
	}
}

func TestProcessRangeRequestRanges(t *testing.T) {
	data := []byte("0123456789")
	serve := func(rangeHeader string) (*httptest.ResponseRecorder, error) {
		r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/test.txt", nil)
		r.Header.Set("Range", rangeHeader)
		w := httptest.NewRecorder()
		err := ProcessRangeRequest(r, w, int64(len(data)), "text/plain", func(offset int64, size int64) (filer.DoStreamContent, error) {
			return func(writer io.Writer) error {
				_, err := writer.Write(data[offset : offset+size])
				return err
			}, nil
		})
		return w, err
	}

	tests := []struct {
		rangeHeader string
		wantCode    int
		wantRange   string
		wantBody    string
	}{
		{"bytes=0-1", http.StatusPartialContent, "bytes 0-1/10", "01"},
		{"bytes=5-100", http.StatusPartialContent, "bytes 5-9/10", "56789"},
		{"bytes=10-", http.StatusRequestedRangeNotSatisfiable, "bytes */10", ""},
		{"bytes=100-", http.StatusRequestedRangeNotSatisfiable, "bytes */10", ""},
		{"bytes=10-,0-1", http.StatusPartialContent, "bytes 0-1/10", "01"},
	}
	for _, tt := range tests {
		w, err := serve(tt.rangeHeader)
		if wantErr := tt.wantCode == http.StatusRequestedRangeNotSatisfiable; (err != nil) != wantErr {
			t.Errorf("%s: error = %v, want an error only for 416", tt.rangeHeader, err)
		}
		if w.Code != tt.wantCode {
			t.Errorf("%s: status %d, want %d", tt.rangeHeader, w.Code, tt.wantCode)
		}
		if got := w.Header().Get("Content-Range"); got != tt.wantRange {
			t.Errorf("%s: Content-Range %q, want %q", tt.rangeHeader, got, tt.wantRange)
		}
		if tt.wantBody != "" && w.Body.String() != tt.wantBody {
			t.Errorf("%s: body %q, want %q", tt.rangeHeader, w.Body.String(), tt.wantBody)
		}
	}
}

func TestProcessRangeRequestIfRange(t *testing.T) {
	data := []byte("0123456789")
	const etag = `"v2"`
	const lastModified = "Sat, 24 Oct 2026 22:01:04 GMT"
	tests := []struct {
		ifRange  string
		wantCode int
		wantBody string
	}{
		{"", http.StatusPartialContent, "01"},
		{etag, http.StatusPartialContent, "01"},
		{lastModified, http.StatusPartialContent, "01"},
		{"Saturday, 24-Oct-26 22:01:04 GMT", http.StatusPartialContent, "01"},
		{"Sat Oct 24 22:01:04 2026", http.StatusPartialContent, "01"},
		{`"v1"`, http.StatusOK, "0123456789"},
		{`W/"v2"`, http.StatusOK, "0123456789"},
		{"Sat, 24 Oct 2026 22:01:03 GMT", http.StatusOK, "0123456789"},
		{"not a date", http.StatusOK, "0123456789"},
	}
	for _, tt := range tests {
		r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/test.txt", nil)
		r.Header.Set("Range", "bytes=0-1")
		if tt.ifRange != "" {
			r.Header.Set("If-Range", tt.ifRange)
		}
		w := httptest.NewRecorder()
		w.Header().Set("ETag", etag)
		w.Header().Set("Last-Modified", lastModified)
		if err := ProcessRangeRequest(r, w, int64(len(data)), "text/plain", func(offset int64, size int64) (filer.DoStreamContent, error) {
			return func(writer io.Writer) error {
				_, err := writer.Write(data[offset : offset+size])
				return err
			}, nil
		}); err != nil {
			t.Errorf("If-Range %q: %v", tt.ifRange, err)
		}
		if w.Code != tt.wantCode || w.Body.String() != tt.wantBody {
			t.Errorf("If-Range %q: got %d %q, want %d %q", tt.ifRange, w.Code, w.Body.String(), tt.wantCode, tt.wantBody)
		}
	}
}

// /submit must reject at the limit its master was given, not at a hardcoded
// 256MB: weed server hands -volume.fileSizeLimitMB down to the master, and an
// upload the volume server would store must not be turned away here (#6748).
func TestSubmitForClientHandlerFileSizeLimit(t *testing.T) {
	const fileSizeLimitBytes = int64(1 << 20)

	submit := func(t *testing.T, dataSize int) *httptest.ResponseRecorder {
		t.Helper()
		var form bytes.Buffer
		mw := multipart.NewWriter(&form)
		part, err := mw.CreateFormFile("file", "test.bin")
		if err != nil {
			t.Fatalf("create form file: %v", err)
		}
		if _, err := part.Write(make([]byte, dataSize)); err != nil {
			t.Fatalf("write form file: %v", err)
		}
		if err := mw.Close(); err != nil {
			t.Fatalf("close multipart writer: %v", err)
		}

		// Assigning a file id is not under test, and there is no master to ask.
		// A cancelled context fails that step at once for an accepted upload.
		ctx, cancel := context.WithCancel(context.Background())
		cancel()
		r := httptest.NewRequestWithContext(ctx, http.MethodPost, "/submit", bytes.NewReader(form.Bytes()))
		r.Header.Set("Content-Type", mw.FormDataContentType())
		w := httptest.NewRecorder()
		masterFn := func(ctx context.Context) pb.ServerAddress { return pb.ServerAddress("localhost:9333") }
		submitForClientHandler(w, r, masterFn, grpc.WithTransportCredentials(insecure.NewCredentials()), fileSizeLimitBytes)
		return w
	}

	t.Run("over the limit", func(t *testing.T) {
		w := submit(t, int(fileSizeLimitBytes)+1)
		if w.Code != http.StatusBadRequest {
			t.Fatalf("status: got %d want %d, body %q", w.Code, http.StatusBadRequest, w.Body.String())
		}
		if !strings.Contains(w.Body.String(), "over the limited") {
			t.Errorf("body: got %q, want the file size limit error", w.Body.String())
		}
	})

	t.Run("under the limit", func(t *testing.T) {
		w := submit(t, int(fileSizeLimitBytes)-1)
		if strings.Contains(w.Body.String(), "over the limited") {
			t.Errorf("body: got %q, want no file size limit error", w.Body.String())
		}
		// Asserting on the message alone would still pass if the limit rejected
		// this payload with different wording. Parser failures answer 400, and the
		// cancelled assignment this request runs into answers 500, so a 400 here
		// means the upload never got past parsing.
		if w.Code == http.StatusBadRequest {
			t.Errorf("status: got 400, want the request to reach assignment, body %q", w.Body.String())
		}
	})
}

func TestRedactedURLDropsTheJwt(t *testing.T) {
	r := httptest.NewRequest(http.MethodGet, "/1,abc?jwt=secret-token&size=5", nil)
	if got := redactedURL(r); strings.Contains(got, "secret-token") || !strings.Contains(got, "size=5") {
		t.Errorf("got %q, want jwt redacted and other params kept", got)
	}
	r = httptest.NewRequest(http.MethodGet, "/1,abc?jwt=&jwt=secret-token&size=5", nil)
	if got := redactedURL(r); strings.Contains(got, "secret-token") {
		t.Errorf("got %q, want every jwt value redacted", got)
	}
	r = httptest.NewRequest(http.MethodGet, "/1,abc?size=5", nil)
	if got := redactedURL(r); got != "/1,abc?size=5" {
		t.Errorf("got %q, want the URL unchanged without a jwt", got)
	}
}