diff --git a/weed/filer/zip_extract.go b/weed/filer/zip_extract.go new file mode 100644 index 000000000..99ed3a624 --- /dev/null +++ b/weed/filer/zip_extract.go @@ -0,0 +1,117 @@ +package filer + +import ( + "archive/zip" + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const ( + // ZipExtractMaxMembers bounds how many members one archive write fans out to. + ZipExtractMaxMembers = 100000 + // ZipExtractMaxBytes bounds the total uncompressed bytes one archive may declare. + ZipExtractMaxBytes = int64(1) << 40 // 1 TiB + + // ZipExtractMaxArchiveBytes bounds the compressed bytes one archive may + // spool to local disk when the caller passes no tighter limit. It matches + // the TUS default max upload size. + ZipExtractMaxArchiveBytes = int64(5) << 30 // 5 GiB + + zipSpoolMemoryLimit = 32 << 20 // 32 MiB; larger archives spool to a temp file +) + +// ErrZipTooLarge marks a spool failure caused by the archive exceeding the +// caller's size limit, so handlers can answer 4xx instead of 5xx. +var ErrZipTooLarge = errors.New("zip archive too large") + +// SpoolZipReader buffers r into a seekable source for zip.NewReader: memory for +// small archives, a temp file otherwise. At most maxArchiveBytes are buffered; +// a non-positive limit falls back to ZipExtractMaxArchiveBytes. The returned +// cleanup releases the spool. +func SpoolZipReader(r io.Reader, maxArchiveBytes int64) (ra io.ReaderAt, n int64, cleanup func(), err error) { + nop := func() {} + if maxArchiveBytes <= 0 { + maxArchiveBytes = ZipExtractMaxArchiveBytes + } + head, err := io.ReadAll(io.LimitReader(r, zipSpoolMemoryLimit+1)) + if err != nil { + return nil, 0, nop, err + } + if int64(len(head)) > maxArchiveBytes { + return nil, 0, nop, fmt.Errorf("%w: exceeds %d bytes", ErrZipTooLarge, maxArchiveBytes) + } + if int64(len(head)) <= zipSpoolMemoryLimit { + return bytes.NewReader(head), int64(len(head)), nop, nil + } + f, err := os.CreateTemp("", "seaweedfs-extract-*.zip") + if err != nil { + return nil, 0, nop, err + } + cleanup = func() { + name := f.Name() + f.Close() + os.Remove(name) + } + if _, err = f.Write(head); err == nil { + var rest int64 + // One byte past the limit so an oversized archive errors out instead + // of being silently truncated. + if rest, err = io.Copy(f, io.LimitReader(r, maxArchiveBytes+1-int64(len(head)))); err == nil { + n = int64(len(head)) + rest + } + } + if err == nil && n > maxArchiveBytes { + err = fmt.Errorf("%w: exceeds %d bytes", ErrZipTooLarge, maxArchiveBytes) + } + if err != nil { + cleanup() + return nil, 0, nop, err + } + return f, n, cleanup, nil +} + +// CheckZipLimits enforces the member-count and total-size caps on an opened archive. +func CheckZipLimits(zr *zip.Reader) error { + if len(zr.File) > ZipExtractMaxMembers { + return fmt.Errorf("zip archive has %d members, limit %d", len(zr.File), ZipExtractMaxMembers) + } + var total uint64 + for _, f := range zr.File { + if f.UncompressedSize64 > uint64(ZipExtractMaxBytes)-total { + return fmt.Errorf("zip archive uncompressed size exceeds %d bytes", ZipExtractMaxBytes) + } + total += f.UncompressedSize64 + } + return nil +} + +// SanitizeZipMemberName converts a zip member name into a relative filer path. +// ok=false means the member must be skipped: absolute or escaping paths, +// non-regular files such as symlinks and devices. +func SanitizeZipMemberName(f *zip.File) (name string, ok bool) { + mode := f.FileInfo().Mode() + if mode&(os.ModeSymlink|os.ModeDevice|os.ModeNamedPipe|os.ModeSocket) != 0 { + return "", false + } + name = strings.TrimSuffix(strings.ReplaceAll(f.Name, "\\", "/"), "/") + if name == "" || strings.HasPrefix(name, "/") { + return "", false + } + for _, seg := range strings.Split(name, "/") { + if seg == "" || seg == "." || seg == ".." || strings.ContainsRune(seg, '\x00') { + return "", false + } + } + // filepath.IsLocal is the check static analysis recognizes: no element may + // be "." or "..", the path may not be absolute or empty, and on Windows it + // may not be a reserved name. + if !filepath.IsLocal(name) { + return "", false + } + return name, true +} diff --git a/weed/filer/zip_extract_test.go b/weed/filer/zip_extract_test.go new file mode 100644 index 000000000..e91b3a635 --- /dev/null +++ b/weed/filer/zip_extract_test.go @@ -0,0 +1,154 @@ +package filer + +import ( + "archive/zip" + "bytes" + "io" + "math" + "os" + "strings" + "testing" + "time" +) + +func zipFile(t *testing.T, name string, mode os.FileMode) *zip.File { + t.Helper() + h := &zip.FileHeader{Name: name, Method: zip.Store, Modified: time.Unix(1700000000, 0)} + h.SetMode(mode) + return &zip.File{FileHeader: *h} +} + +func TestSanitizeZipMemberName(t *testing.T) { + tests := []struct { + name string + mode os.FileMode + want string + ok bool + }{ + {"a.png", 0644, "a.png", true}, + {"dir/b.txt", 0644, "dir/b.txt", true}, + {"dir\\b.txt", 0644, "dir/b.txt", true}, + {"dir/", os.ModeDir | 0755, "dir", true}, + {"../escape.txt", 0644, "", false}, + {"a/../b.txt", 0644, "", false}, + {"./dot.txt", 0644, "", false}, + {"/abs.txt", 0644, "", false}, + {"a//b.txt", 0644, "", false}, + {"", 0644, "", false}, + {"link.txt", os.ModeSymlink | 0777, "", false}, + {"dev", os.ModeDevice | 0666, "", false}, + } + for _, tt := range tests { + got, ok := SanitizeZipMemberName(zipFile(t, tt.name, tt.mode)) + if ok != tt.ok || got != tt.want { + t.Errorf("SanitizeZipMemberName(%q, %v) = (%q, %v), want (%q, %v)", tt.name, tt.mode, got, ok, tt.want, tt.ok) + } + } +} + +func TestSpoolZipReaderMemory(t *testing.T) { + data := []byte(strings.Repeat("x", 1024)) + ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0) + if err != nil { + t.Fatalf("spool: %v", err) + } + defer cleanup() + if n != int64(len(data)) { + t.Fatalf("size = %d, want %d", n, len(data)) + } + if _, isFile := ra.(*os.File); isFile { + t.Fatal("small archive should spool to memory") + } + buf := make([]byte, 4) + if _, err := ra.ReadAt(buf, 0); err != nil || string(buf) != "xxxx" { + t.Fatalf("ReadAt = %q, %v", buf, err) + } +} + +func TestSpoolZipReaderTempFile(t *testing.T) { + data := bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1) + ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0) + if err != nil { + t.Fatalf("spool: %v", err) + } + f, isFile := ra.(*os.File) + if !isFile { + cleanup() + t.Fatal("large archive should spool to a temp file") + } + name := f.Name() + cleanup() + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Fatalf("temp file %s still exists after cleanup: %v", name, err) + } + if n != int64(len(data)) { + t.Fatalf("size = %d, want %d", n, len(data)) + } +} + +func TestSpoolZipReaderLimit(t *testing.T) { + data := []byte(strings.Repeat("x", 2048)) + if _, _, _, err := SpoolZipReader(bytes.NewReader(data), 1024); err == nil { + t.Fatal("expected limit error for in-memory archive") + } + if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1024)), zipSpoolMemoryLimit+512); err == nil { + t.Fatal("expected limit error for spooled archive") + } else { + cleanup() + } +} + +func TestCheckZipLimits(t *testing.T) { + over := &zip.Reader{File: make([]*zip.File, ZipExtractMaxMembers+1)} + for i := range over.File { + over.File[i] = &zip.File{} + } + if err := CheckZipLimits(over); err == nil { + t.Fatal("expected member count limit error") + } + + big := &zip.Reader{File: []*zip.File{{}}} + big.File[0].UncompressedSize64 = uint64(ZipExtractMaxBytes) + 1 + if err := CheckZipLimits(big); err == nil { + t.Fatal("expected total size limit error") + } + + overflow := &zip.Reader{File: []*zip.File{{}, {}}} + overflow.File[0].UncompressedSize64 = math.MaxUint64 + overflow.File[1].UncompressedSize64 = math.MaxUint64 + if err := CheckZipLimits(overflow); err == nil { + t.Fatal("expected total size overflow error") + } + + ok := &zip.Reader{File: []*zip.File{{}, {}}} + if err := CheckZipLimits(ok); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestSpoolZipReaderRoundTrip(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("a/b.txt") + w.Write([]byte("hello")) + zw.Close() + + ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(buf.Bytes()), 0) + if err != nil { + t.Fatalf("spool: %v", err) + } + defer cleanup() + zr, err := zip.NewReader(ra, n) + if err != nil { + t.Fatalf("open zip: %v", err) + } + rc, err := zr.File[0].Open() + if err != nil { + t.Fatalf("open member: %v", err) + } + defer rc.Close() + data, _ := io.ReadAll(rc) + if string(data) != "hello" { + t.Fatalf("member content = %q", data) + } +} diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 7f03adbb3..f88fc351b 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -58,6 +58,7 @@ const ( AmzDeleteMarker = "X-Amz-Delete-Marker" SeaweedFSUploadId = "X-Seaweedfs-Upload-Id" + SeaweedFSExtract = "X-Seaweedfs-Extract" SeaweedFSMultipartPartsCount = "X-Seaweedfs-Multipart-Parts-Count" SeaweedFSMultipartPartBoundaries = "X-Seaweedfs-Multipart-Part-Boundaries" // JSON: [{part:1,start:0,end:2,etag:"abc"},{part:2,start:2,end:3,etag:"def"}] SeaweedFSExpiresS3 = "X-Seaweedfs-Expires-S3" diff --git a/weed/server/filer_server_handlers_extract.go b/weed/server/filer_server_handlers_extract.go new file mode 100644 index 000000000..c51d7b9c6 --- /dev/null +++ b/weed/server/filer_server_handlers_extract.go @@ -0,0 +1,339 @@ +package weed_server + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/util/constants" +) + +type FilerExtractResult struct { + Name string `json:"name,omitempty"` + Extracted []*FilerPostResult `json:"extracted,omitempty"` + Errors []string `json:"errors,omitempty"` +} + +func isExtractZip(r *http.Request) bool { + return r.URL.Query().Get("extract") == "zip" || + r.Header.Get(s3_constants.SeaweedFSExtract) == "zip" +} + +// extractZipArchive writes a posted archive's members as individual entries +// under the target directory, like AWS Snowball's auto-extract. The archive +// itself is only stored when ?extract.keep=true. +func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWriter, r *http.Request) { + var reader io.Reader = r.Body + archiveName := path.Base(r.URL.Path) + if r.Method == http.MethodPost { + if mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); strings.HasPrefix(mediaType, "multipart/") { + multipartReader, multipartReaderErr := r.MultipartReader() + if multipartReaderErr != nil { + writeJsonError(w, r, http.StatusBadRequest, multipartReaderErr) + return + } + part1, part1Err := multipartReader.NextPart() + if part1Err != nil { + writeJsonError(w, r, http.StatusBadRequest, part1Err) + return + } + defer part1.Close() + reader = part1 + if fileName := part1.FileName(); fileName != "" { + archiveName = path.Base(fileName) + } + } + } + + ra, size, cleanup, err := filer.SpoolZipReader(reader, fs.option.TusMaxSize) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, filer.ErrZipTooLarge) { + code = http.StatusRequestEntityTooLarge + } + writeJsonError(w, r, code, fmt.Errorf("read input: %w", err)) + return + } + defer cleanup() + zr, err := zip.NewReader(ra, size) + if err != nil { + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("extract zip: %w", err)) + return + } + if err = filer.CheckZipLimits(zr); err != nil { + writeJsonError(w, r, http.StatusBadRequest, err) + return + } + + root := r.URL.Path + if !strings.HasSuffix(root, "/") { + root = path.Dir(root) + } + root = strings.TrimSuffix(root, "/") + + query := r.URL.Query() + chunkSize := fs.requestChunkSize(query) + + // x-seaweedfs-destination picks storage rules for a logical destination + // while members land under the request path, mirroring PostHandler. The + // actual write path keeps its own read-only enforcement in + // saveExtractedEntry and mkdirExtracted. + ruleRoot := root + if headerDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); headerDestination != "" { + ruleRoot = strings.TrimSuffix(headerDestination, "/") + } + + // Members may land under storage rules different from the request path; + // resolve options per destination, keeping the request's explicit overrides. + memberStorageOption := func(rulePath, writePath string) (*operation.StorageOption, error) { + memberSO, detectErr := fs.detectStorageOption0(ctx, rulePath, + query.Get("collection"), + query.Get("replication"), + query.Get("ttl"), + query.Get("disk"), + query.Get("fsync"), + query.Get("dataCenter"), + query.Get("rack"), + query.Get("dataNode"), + query.Get("saveInside"), + ) + if detectErr != nil { + return nil, detectErr + } + if memberSO.DiskType == "" { + memberSO.DiskType = fs.option.DiskType + } + if strings.HasPrefix(writePath, "/etc") { + memberSO.SaveInside = true + } + return memberSO, nil + } + + result := &FilerExtractResult{Name: archiveName} + writeMember := func(rel string, memberReader io.Reader, memberSize int64, mtime time.Time, mode os.FileMode) { + destPath := root + "/" + rel + memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath) + if soErr != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr)) + return + } + memberResult, memberErr := fs.saveExtractedEntry(ctx, destPath, memberReader, memberSize, path.Base(rel), chunkSize, memberSO, mtime, mode) + if memberErr != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, memberErr)) + return + } + memberResult.Name = rel + result.Extracted = append(result.Extracted, memberResult) + } + mkdirMember := func(rel string, zf *zip.File) { + destPath := root + "/" + rel + memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath) + if soErr != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr)) + return + } + if dirErr := fs.mkdirExtracted(ctx, destPath, zf, memberSO); dirErr != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, dirErr)) + } + } + + if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != "/" { + mode := os.FileMode(0660) + if modeStr := query.Get("mode"); modeStr != "" { + if parsed, parseErr := strconv.ParseUint(modeStr, 8, 32); parseErr == nil { + mode = os.FileMode(parsed) + } + } + writeMember(archiveName, io.NewSectionReader(ra, 0, size), size, time.Now(), mode) + } + + // Directory members first so their stored metadata is not overwritten by + // parents auto-created for an earlier file member. + for _, dirsOnly := range []bool{true, false} { + for _, zf := range zr.File { + if ctxErr := ctx.Err(); ctxErr != nil { + result.Errors = append(result.Errors, ctxErr.Error()) + writeJsonQuiet(w, r, http.StatusCreated, result) + return + } + rel, ok := filer.SanitizeZipMemberName(zf) + if !ok { + if !dirsOnly { + result.Errors = append(result.Errors, fmt.Sprintf("%s: unsafe member name", zf.Name)) + } + continue + } + if zf.FileInfo().IsDir() != dirsOnly { + continue + } + if dirsOnly { + mkdirMember(rel, zf) + continue + } + memberReader, openErr := zf.Open() + if openErr != nil { + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, openErr)) + continue + } + mode := os.FileMode(0660) + if perm := zf.Mode().Perm(); perm != 0 { + mode = perm + } + writeMember(rel, memberReader, int64(zf.UncompressedSize64), zf.Modified, mode) + memberReader.Close() + } + } + + writeJsonQuiet(w, r, http.StatusCreated, result) +} + +func (fs *FilerServer) mkdirExtracted(ctx context.Context, destPath string, zf *zip.File, so *operation.StorageOption) error { + if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly { + return ErrReadOnly + } + existing, findErr := fs.filer.FindEntry(ctx, util.FullPath(destPath)) + if findErr == nil { + if existing.IsDirectory() { + return nil + } + return fmt.Errorf("%s already exists as a file", destPath) + } + if !errors.Is(findErr, filer_pb.ErrNotFound) { + return findErr + } + mode := os.FileMode(0660) + if perm := zf.Mode().Perm(); perm != 0 { + mode = perm + } + mtime := zf.Modified + if mtime.IsZero() { + mtime = time.Now() + } + return fs.filer.CreateEntry(context.WithoutCancel(ctx), &filer.Entry{ + FullPath: util.FullPath(destPath), + Attr: filer.Attr{ + Mtime: mtime, + Crtime: time.Now(), + Mode: mode | os.ModeDir, + Uid: OS_UID, + Gid: OS_GID, + TtlSec: so.TtlSeconds, + }, + }, nil, false, false, nil, false, so.MaxFileNameLength) +} + +func (fs *FilerServer) saveExtractedEntry(ctx context.Context, destPath string, reader io.Reader, memberSize int64, fileName string, chunkSize int32, so *operation.StorageOption, mtime time.Time, mode os.FileMode) (*FilerPostResult, error) { + if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly { + return nil, ErrReadOnly + } + wormEnforced, err := fs.wormEnforcedForEntry(ctx, destPath) + if err != nil { + return nil, err + } + if wormEnforced { + return nil, errors.New(constants.ErrMsgOperationNotPermitted) + } + if util.FullPath(destPath).IsLongerFileName(so.MaxFileNameLength) { + return nil, filer_pb.ErrEntryNameTooLong + } + + contentType := mime.TypeByExtension(path.Ext(fileName)) + + // saveInside stores members as inline entry content like a normal + // multipart upload; the declared size keeps a corrupt archive from + // buffering more than it advertised. + if so.SaveInside { + if memberSize < 0 || memberSize > filer.ZipExtractMaxBytes { + return nil, fmt.Errorf("zip member %s size %d out of bounds", fileName, memberSize) + } + var buf bytes.Buffer + if _, err := buf.ReadFrom(io.LimitReader(reader, memberSize+1)); err != nil { + return nil, fmt.Errorf("read member: %w", err) + } + if int64(buf.Len()) > memberSize { + return nil, fmt.Errorf("zip member %s is larger than its declared size", fileName) + } + if mtime.IsZero() { + mtime = time.Now() + } + entry := &filer.Entry{ + FullPath: util.FullPath(destPath), + Attr: filer.Attr{ + Mtime: mtime, + Crtime: time.Now(), + Mode: mode, + Uid: OS_UID, + Gid: OS_GID, + TtlSec: so.TtlSeconds, + Mime: contentType, + Md5: util.Md5(buf.Bytes()), + FileSize: uint64(buf.Len()), + }, + Content: buf.Bytes(), + } + if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil { + return nil, err + } + return &FilerPostResult{Name: fileName, Size: int64(buf.Len())}, nil + } + + fileChunks, md5Hash, chunkOffset, uploadErr, smallContent := fs.uploadReaderToChunks(ctx, nil, reader, 0, chunkSize, fileName, contentType, false, so) + if uploadErr != nil { + return nil, uploadErr + } + if chunkOffset == 0 { + smallContent = []byte{} + } + + mergedChunks, mergeErr := fs.maybeMergeChunks(ctx, so, fileChunks) + if mergeErr != nil { + glog.V(0).InfofCtx(ctx, "merge chunks %s: %v", destPath, mergeErr) + mergedChunks = fileChunks + } + mergedChunks, manifestErr := filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, mergedChunks) + if manifestErr != nil { + fs.filer.DeleteUncommittedChunks(ctx, mergedChunks) + return nil, manifestErr + } + + if mtime.IsZero() { + mtime = time.Now() + } + entry := &filer.Entry{ + FullPath: util.FullPath(destPath), + Attr: filer.Attr{ + Mtime: mtime, + Crtime: time.Now(), + Mode: mode, + Uid: OS_UID, + Gid: OS_GID, + TtlSec: so.TtlSeconds, + Mime: contentType, + Md5: md5Hash.Sum(nil), + FileSize: uint64(chunkOffset), + }, + Content: smallContent, + Chunks: mergedChunks, + } + if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil { + fs.filer.DeleteUncommittedChunks(ctx, mergedChunks) + return nil, err + } + return &FilerPostResult{Name: fileName, Size: chunkOffset}, nil +} diff --git a/weed/server/filer_server_handlers_extract_test.go b/weed/server/filer_server_handlers_extract_test.go new file mode 100644 index 000000000..9461e79e5 --- /dev/null +++ b/weed/server/filer_server_handlers_extract_test.go @@ -0,0 +1,274 @@ +package weed_server + +import ( + "archive/zip" + "bytes" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func buildTestZip(t *testing.T, members map[string]string, dirs []string, unsafe []string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, dir := range dirs { + h := &zip.FileHeader{Name: dir, Method: zip.Store, Modified: time.Unix(1700000000, 0)} + h.SetMode(0755 | 1<<31) + if _, err := zw.CreateHeader(h); err != nil { + t.Fatalf("create dir %s: %v", dir, err) + } + } + write := func(name, content string) { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("create %s: %v", name, err) + } + if _, err := w.Write([]byte(content)); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + for name, content := range members { + write(name, content) + } + for _, name := range unsafe { + write(name, "evil") + } + if err := zw.Close(); err != nil { + t.Fatalf("close zip: %v", err) + } + return buf.Bytes() +} + +func newExtractTestServer(t *testing.T) (*FilerServer, *filer.Filer) { + store := newRenameTestStore() + f := newRenameTestFiler(t, store) + return &FilerServer{ + filer: f, + option: &FilerOption{ + MaxMB: 1, + SaveToFilerLimit: 1 << 20, + }, + }, f +} + +func TestExtractZipArchivePut(t *testing.T) { + fs, f := newExtractTestServer(t) + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + dirHeader := &zip.FileHeader{Name: "empty/", Method: zip.Store, Modified: time.Unix(1700000000, 0)} + dirHeader.SetMode(0755 | 1<<31) + if _, err := zw.CreateHeader(dirHeader); err != nil { + t.Fatal(err) + } + fileHeader := &zip.FileHeader{Name: "a/b.txt", Modified: time.Unix(1600000000, 0)} + fileHeader.SetMode(0600) + fw, err := zw.CreateHeader(fileHeader) + if err != nil { + t.Fatal(err) + } + if _, err := fw.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + pngW, err := zw.Create("c.png") + if err != nil { + t.Fatal(err) + } + if _, err := pngW.Write([]byte("\x89png")); err != nil { + t.Fatal(err) + } + evil, err := zw.Create("../evil.txt") + if err != nil { + t.Fatal(err) + } + if _, err := evil.Write([]byte("evil")); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + zipBytes := buf.Bytes() + + r := httptest.NewRequest(http.MethodPut, "/dst/archive.zip?extract=zip", bytes.NewReader(zipBytes)) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201, body: %s", w.Code, w.Body.String()) + } + var result FilerExtractResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if len(result.Extracted) != 2 { + t.Fatalf("extracted %d members, want 2: %+v", len(result.Extracted), result) + } + if len(result.Errors) != 1 { + t.Fatalf("errors = %v, want 1 unsafe-name entry", result.Errors) + } + + entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/a/b.txt")) + if err != nil { + t.Fatalf("find extracted member: %v", err) + } + if string(entry.Content) != "hello" { + t.Fatalf("member content = %q", entry.Content) + } + if entry.Mtime.Unix() != 1600000000 { + t.Fatalf("member mtime = %v, want %v", entry.Mtime, time.Unix(1600000000, 0)) + } + if entry.Mode.Perm() != 0600 { + t.Fatalf("member mode = %v, want 0600", entry.Mode.Perm()) + } + + dirEntry, err := f.FindEntry(r.Context(), util.FullPath("/dst/empty")) + if err != nil || !dirEntry.IsDirectory() { + t.Fatalf("dir member missing: %v %+v", err, dirEntry) + } + if dirEntry.Mtime.Unix() != 1700000000 { + t.Fatalf("dir mtime = %v, want %v", dirEntry.Mtime, time.Unix(1700000000, 0)) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/archive.zip")); err == nil { + t.Fatal("archive should not be stored without extract.keep") + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/../evil.txt")); err == nil { + t.Fatal("unsafe member should not be extracted") + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/evil.txt")); err == nil { + t.Fatal("zip-slip member escaped the extraction root") + } +} + +func TestExtractZipArchiveKeep(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"x.txt": "data"}, nil, nil) + + r := httptest.NewRequest(http.MethodPut, "/dst/pack.zip?extract=zip&extract.keep=true", bytes.NewReader(zipBytes)) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + archive, err := f.FindEntry(r.Context(), util.FullPath("/dst/pack.zip")) + if err != nil { + t.Fatalf("archive not stored with extract.keep: %v", err) + } + if int64(len(archive.Content)) != int64(len(zipBytes)) { + t.Fatalf("archive size = %d, want %d", len(archive.Content), len(zipBytes)) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/x.txt")); err != nil { + t.Fatalf("member missing: %v", err) + } +} + +func TestExtractZipArchiveNotAZip(t *testing.T) { + fs, _ := newExtractTestServer(t) + r := httptest.NewRequest(http.MethodPut, "/dst/notzip.bin?extract=zip", bytes.NewReader([]byte("not a zip"))) + w := httptest.NewRecorder() + fs.PostHandler(w, r, 9) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", w.Code) + } +} + +// A member larger than chunkSize must not take the inline fast path when +// SaveToFilerLimit exceeds chunkSize — the first chunk is not the whole member. +func TestExtractZipArchiveMemberLargerThanChunk(t *testing.T) { + store := newRenameTestStore() + f := newRenameTestFiler(t, store) + fs, _ := newInlineAppendUploadServer(t, f, 4<<20, 1) // inline limit exceeds the 1 MiB chunkSize + member := bytes.Repeat([]byte("z"), (1<<20)+1) + zipBytes := buildTestZip(t, map[string]string{"big.bin": string(member)}, nil, nil) + + r := httptest.NewRequest(http.MethodPut, "/dst/pack.zip?extract=zip", bytes.NewReader(zipBytes)) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + var result FilerExtractResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if len(result.Errors) != 0 { + t.Fatalf("member errors: %v", result.Errors) + } + entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/big.bin")) + if err != nil { + t.Fatalf("member missing: %v", err) + } + if int64(entry.Attr.FileSize) != int64(len(member)) { + t.Fatalf("member stored truncated: %d bytes, want %d", entry.Attr.FileSize, len(member)) + } +} + +func TestExtractZipArchiveRawPost(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"raw.txt": "raw"}, nil, nil) + + r := httptest.NewRequest(http.MethodPost, "/dst/?extract=zip", bytes.NewReader(zipBytes)) + r.Header.Set("Content-Type", "application/zip") + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/raw.txt")) + if err != nil { + t.Fatalf("member missing: %v", err) + } + if string(entry.Content) != "raw" { + t.Fatalf("member content = %q", entry.Content) + } +} + +func TestExtractZipArchiveMultipartPost(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"img/1.png": "png-bytes"}, nil, nil) + + var body bytes.Buffer + mw := multipartWriter(t, &body, "bundle.zip", zipBytes) + r := httptest.NewRequest(http.MethodPost, "/assets/?extract=zip", &body) + r.Header.Set("Content-Type", mw) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(body.Len())) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + entry, err := f.FindEntry(r.Context(), util.FullPath("/assets/img/1.png")) + if err != nil { + t.Fatalf("member missing: %v", err) + } + if string(entry.Content) != "png-bytes" { + t.Fatalf("member content = %q", entry.Content) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/assets/bundle.zip")); err == nil { + t.Fatal("archive should not be stored") + } +} + +func multipartWriter(t *testing.T, buf *bytes.Buffer, fileName string, data []byte) string { + t.Helper() + mw := multipart.NewWriter(buf) + part, err := mw.CreateFormFile("file", fileName) + if err != nil { + t.Fatalf("create form file: %v", err) + } + if _, err := part.Write(data); err != nil { + t.Fatalf("write part: %v", err) + } + if err := mw.Close(); err != nil { + t.Fatalf("close multipart: %v", err) + } + return mw.FormDataContentType() +} diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index b1a96e07c..4bb91c9d0 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -141,6 +141,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte fs.move(ctx, w, r, so) } else if query.Has("cp.from") { fs.copy(ctx, w, r, so) + } else if isExtractZip(r) { + fs.extractZipArchive(ctx, w, r) } else { fs.autoChunk(ctx, w, r, contentLength, so) } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index a2de8e74f..f6df863eb 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "io" + "math" "net/http" + "net/url" "os" "path" "strconv" @@ -29,13 +31,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line query := r.URL.Query() - parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) - maxMB := int32(parsedMaxMB) - if maxMB <= 0 && fs.option.MaxMB > 0 { - maxMB = int32(fs.option.MaxMB) - } - - chunkSize := 1024 * 1024 * maxMB + chunkSize := fs.requestChunkSize(query) var reply *FilerPostResult var err error @@ -178,6 +174,22 @@ func skipCheckParentDirEntry(r *http.Request) bool { return r.URL.Query().Get("skipCheckParentDir") == "true" } +// requestChunkSize resolves the auto-chunking size from the maxMB query param +// or the server default, clamped so the int32 chunk size cannot overflow into +// a non-positive value that would drop the upload's contents. +func (fs *FilerServer) requestChunkSize(query url.Values) int32 { + parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) + maxMB := parsedMaxMB + if maxMB <= 0 { + maxMB = int64(fs.option.MaxMB) + } + const maxChunkSizeMB = math.MaxInt32 / (1024 * 1024) // 2047 + if maxMB > maxChunkSizeMB { + maxMB = maxChunkSizeMB + } + return int32(maxMB) * 1024 * 1024 +} + func (fs *FilerServer) checkPermissions(ctx context.Context, r *http.Request, fileName string) error { fullPath := fs.fixFilePath(ctx, r, fileName) enforced, err := fs.wormEnforcedForEntry(ctx, fullPath) diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index f41ba6151..5631e3e55 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -149,7 +149,9 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request break } if chunkOffset == 0 && !isAppend { - if dataSize < fs.option.SaveToFilerLimit { + // A short first read proves EOF, so the buffer holds the whole file. + // A full chunk may have more data behind it — store it chunked. + if dataSize < fs.option.SaveToFilerLimit && dataSize < int64(chunkSize) { chunkOffset += dataSize smallContent = make([]byte, dataSize) bytesBuffer.Read(smallContent)