seaweedfs/weed/server/filer_server_handlers_extract.go

package weed_server

import (
	"archive/zip"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"mime"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/seaweedfs/seaweedfs/weed/util/constants"
)

type FilerExtractResult struct {
	Name      string             `json:"name,omitempty"`
	Extracted []*FilerPostResult `json:"extracted,omitempty"`
	Errors    []string           `json:"errors,omitempty"`
}

func isExtractZip(r *http.Request) bool {
	return r.URL.Query().Get("extract") == "zip" ||
		r.Header.Get(s3_constants.SeaweedFSExtract) == "zip"
}

// extractZipArchive writes a posted archive's members as individual entries
// under the target directory, like AWS Snowball's auto-extract. The archive
// itself is only stored when ?extract.keep=true.
func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	var reader io.Reader = r.Body
	archiveName := path.Base(r.URL.Path)
	if r.Method == http.MethodPost {
		if mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); strings.HasPrefix(mediaType, "multipart/") {
			multipartReader, multipartReaderErr := r.MultipartReader()
			if multipartReaderErr != nil {
				writeJsonError(w, r, http.StatusBadRequest, multipartReaderErr)
				return
			}
			part1, part1Err := multipartReader.NextPart()
			if part1Err != nil {
				writeJsonError(w, r, http.StatusBadRequest, part1Err)
				return
			}
			defer part1.Close()
			reader = part1
			if fileName := part1.FileName(); fileName != "" {
				archiveName = path.Base(fileName)
			}
		}
	}

	ra, size, cleanup, err := filer.SpoolZipReader(reader, fs.option.TusMaxSize)
	if err != nil {
		code := http.StatusInternalServerError
		if errors.Is(err, filer.ErrZipTooLarge) {
			code = http.StatusRequestEntityTooLarge
		}
		writeJsonError(w, r, code, fmt.Errorf("read input: %w", err))
		return
	}
	defer cleanup()
	zr, err := zip.NewReader(ra, size)
	if err != nil {
		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("extract zip: %w", err))
		return
	}
	if err = filer.CheckZipLimits(zr); err != nil {
		writeJsonError(w, r, http.StatusBadRequest, err)
		return
	}

	root := r.URL.Path
	if !strings.HasSuffix(root, "/") {
		root = path.Dir(root)
	}
	root = strings.TrimSuffix(root, "/")

	query := r.URL.Query()
	chunkSize := fs.requestChunkSize(query)

	// x-seaweedfs-destination picks storage rules for a logical destination
	// while members land under the request path, mirroring PostHandler. The
	// actual write path keeps its own read-only enforcement in
	// saveExtractedEntry and mkdirExtracted.
	ruleRoot := root
	if headerDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); headerDestination != "" {
		ruleRoot = strings.TrimSuffix(headerDestination, "/")
	}

	// Members may land under storage rules different from the request path;
	// resolve options per destination, keeping the request's explicit overrides.
	memberStorageOption := func(rulePath, writePath string) (*operation.StorageOption, error) {
		memberSO, detectErr := fs.detectStorageOption0(ctx, rulePath,
			query.Get("collection"),
			query.Get("replication"),
			query.Get("ttl"),
			query.Get("disk"),
			query.Get("fsync"),
			query.Get("dataCenter"),
			query.Get("rack"),
			query.Get("dataNode"),
			query.Get("saveInside"),
		)
		if detectErr != nil {
			return nil, detectErr
		}
		if memberSO.DiskType == "" {
			memberSO.DiskType = fs.option.DiskType
		}
		if strings.HasPrefix(writePath, "/etc") {
			memberSO.SaveInside = true
		}
		return memberSO, nil
	}

	result := &FilerExtractResult{Name: archiveName}
	writeMember := func(rel string, memberReader io.Reader, memberSize int64, mtime time.Time, mode os.FileMode) {
		destPath := root + "/" + rel
		memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath)
		if soErr != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr))
			return
		}
		memberResult, memberErr := fs.saveExtractedEntry(ctx, destPath, memberReader, memberSize, path.Base(rel), chunkSize, memberSO, mtime, mode)
		if memberErr != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, memberErr))
			return
		}
		memberResult.Name = rel
		result.Extracted = append(result.Extracted, memberResult)
	}
	mkdirMember := func(rel string, zf *zip.File) {
		destPath := root + "/" + rel
		memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath)
		if soErr != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr))
			return
		}
		if dirErr := fs.mkdirExtracted(ctx, destPath, zf, memberSO); dirErr != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, dirErr))
		}
	}

	if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != "/" {
		mode := os.FileMode(0660)
		if modeStr := query.Get("mode"); modeStr != "" {
			if parsed, parseErr := strconv.ParseUint(modeStr, 8, 32); parseErr == nil {
				mode = os.FileMode(parsed)
			}
		}
		writeMember(archiveName, io.NewSectionReader(ra, 0, size), size, time.Now(), mode)
	}

	// Directory members first so their stored metadata is not overwritten by
	// parents auto-created for an earlier file member.
	for _, dirsOnly := range []bool{true, false} {
		for _, zf := range zr.File {
			if ctxErr := ctx.Err(); ctxErr != nil {
				result.Errors = append(result.Errors, ctxErr.Error())
				writeJsonQuiet(w, r, http.StatusCreated, result)
				return
			}
			rel, ok := filer.SanitizeZipMemberName(zf)
			if !ok {
				if !dirsOnly {
					result.Errors = append(result.Errors, fmt.Sprintf("%s: unsafe member name", zf.Name))
				}
				continue
			}
			if zf.FileInfo().IsDir() != dirsOnly {
				continue
			}
			if dirsOnly {
				mkdirMember(rel, zf)
				continue
			}
			memberReader, openErr := zf.Open()
			if openErr != nil {
				result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, openErr))
				continue
			}
			mode := os.FileMode(0660)
			if perm := zf.Mode().Perm(); perm != 0 {
				mode = perm
			}
			writeMember(rel, memberReader, int64(zf.UncompressedSize64), zf.Modified, mode)
			memberReader.Close()
		}
	}

	writeJsonQuiet(w, r, http.StatusCreated, result)
}

func (fs *FilerServer) mkdirExtracted(ctx context.Context, destPath string, zf *zip.File, so *operation.StorageOption) error {
	if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly {
		return ErrReadOnly
	}
	existing, findErr := fs.filer.FindEntry(ctx, util.FullPath(destPath))
	if findErr == nil {
		if existing.IsDirectory() {
			return nil
		}
		return fmt.Errorf("%s already exists as a file", destPath)
	}
	if !errors.Is(findErr, filer_pb.ErrNotFound) {
		return findErr
	}
	mode := os.FileMode(0660)
	if perm := zf.Mode().Perm(); perm != 0 {
		mode = perm
	}
	mtime := zf.Modified
	if mtime.IsZero() {
		mtime = time.Now()
	}
	return fs.filer.CreateEntry(context.WithoutCancel(ctx), &filer.Entry{
		FullPath: util.FullPath(destPath),
		Attr: filer.Attr{
			Mtime:  mtime,
			Crtime: time.Now(),
			Mode:   mode | os.ModeDir,
			Uid:    OS_UID,
			Gid:    OS_GID,
			TtlSec: so.TtlSeconds,
		},
	}, nil, false, false, nil, false, so.MaxFileNameLength)
}

func (fs *FilerServer) saveExtractedEntry(ctx context.Context, destPath string, reader io.Reader, memberSize int64, fileName string, chunkSize int32, so *operation.StorageOption, mtime time.Time, mode os.FileMode) (*FilerPostResult, error) {
	if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly {
		return nil, ErrReadOnly
	}
	wormEnforced, err := fs.wormEnforcedForEntry(ctx, destPath)
	if err != nil {
		return nil, err
	}
	if wormEnforced {
		return nil, errors.New(constants.ErrMsgOperationNotPermitted)
	}
	if util.FullPath(destPath).IsLongerFileName(so.MaxFileNameLength) {
		return nil, filer_pb.ErrEntryNameTooLong
	}

	contentType := mime.TypeByExtension(path.Ext(fileName))

	// saveInside stores members as inline entry content like a normal
	// multipart upload; the declared size keeps a corrupt archive from
	// buffering more than it advertised.
	if so.SaveInside {
		if memberSize < 0 || memberSize > filer.ZipExtractMaxBytes {
			return nil, fmt.Errorf("zip member %s size %d out of bounds", fileName, memberSize)
		}
		var buf bytes.Buffer
		if _, err := buf.ReadFrom(io.LimitReader(reader, memberSize+1)); err != nil {
			return nil, fmt.Errorf("read member: %w", err)
		}
		if int64(buf.Len()) > memberSize {
			return nil, fmt.Errorf("zip member %s is larger than its declared size", fileName)
		}
		if mtime.IsZero() {
			mtime = time.Now()
		}
		entry := &filer.Entry{
			FullPath: util.FullPath(destPath),
			Attr: filer.Attr{
				Mtime:    mtime,
				Crtime:   time.Now(),
				Mode:     mode,
				Uid:      OS_UID,
				Gid:      OS_GID,
				TtlSec:   so.TtlSeconds,
				Mime:     contentType,
				Md5:      util.Md5(buf.Bytes()),
				FileSize: uint64(buf.Len()),
			},
			Content: buf.Bytes(),
		}
		if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil {
			return nil, err
		}
		return &FilerPostResult{Name: fileName, Size: int64(buf.Len())}, nil
	}

	fileChunks, md5Hash, chunkOffset, uploadErr, smallContent := fs.uploadReaderToChunks(ctx, nil, reader, 0, chunkSize, fileName, contentType, false, so)
	if uploadErr != nil {
		return nil, uploadErr
	}
	if chunkOffset == 0 {
		smallContent = []byte{}
	}

	mergedChunks, mergeErr := fs.maybeMergeChunks(ctx, so, fileChunks)
	if mergeErr != nil {
		glog.V(0).InfofCtx(ctx, "merge chunks %s: %v", destPath, mergeErr)
		mergedChunks = fileChunks
	}
	mergedChunks, manifestErr := filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, mergedChunks)
	if manifestErr != nil {
		fs.filer.DeleteUncommittedChunks(ctx, mergedChunks)
		return nil, manifestErr
	}

	if mtime.IsZero() {
		mtime = time.Now()
	}
	entry := &filer.Entry{
		FullPath: util.FullPath(destPath),
		Attr: filer.Attr{
			Mtime:    mtime,
			Crtime:   time.Now(),
			Mode:     mode,
			Uid:      OS_UID,
			Gid:      OS_GID,
			TtlSec:   so.TtlSeconds,
			Mime:     contentType,
			Md5:      md5Hash.Sum(nil),
			FileSize: uint64(chunkOffset),
		},
		Content: smallContent,
		Chunks:  mergedChunks,
	}
	if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil {
		fs.filer.DeleteUncommittedChunks(ctx, mergedChunks)
		return nil, err
	}
	return &FilerPostResult{Name: fileName, Size: chunkOffset}, nil
}