diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index cb1e012e3d..107beb333d 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1618,7 +1618,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP output.IsTruncated = aws.Bool(false) return output, s3err.ErrNone } - entries, isLast, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts)) + numbers, latest, truncated, err := s3a.listPartAttempts(input) if err != nil { // A store that reports the missing upload directory as not-found means // the upload is gone (completed or aborted), not a store error. @@ -1632,33 +1632,22 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP // Note: The upload directory is sort of a marker of the existence of an multipart upload request. // So can not just delete empty upload folders. - output.IsTruncated = aws.Bool(!isLast) + output.IsTruncated = aws.Bool(truncated) - for _, entry := range entries { - if strings.HasSuffix(entry.Name, multipartExt) && !entry.IsDirectory { - partNumber, err := parsePartNumber(entry.Name) - if err != nil { - glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err) - continue - } - // the marker is exclusive even if a store ignores startFrom - if partNumber <= int(*input.PartNumberMarker) { - continue - } - - partETag := getEtagFromEntry(entry) - part := &s3.Part{ - PartNumber: aws.Int64(int64(partNumber)), - LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()), - Size: aws.Int64(int64(filer.FileSize(entry))), - ETag: aws.String(partETag), - } - output.Part = append(output.Part, part) - glog.V(3).Infof("listObjectParts: Added part %d, size=%d, etag=%s", - partNumber, filer.FileSize(entry), partETag) - if !isLast { - output.NextPartNumberMarker = aws.Int64(int64(partNumber)) - } + for _, partNumber := range numbers { + entry := latest[partNumber] + partETag := getEtagFromEntry(entry) + part := &s3.Part{ + PartNumber: aws.Int64(int64(partNumber)), + LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()), + Size: aws.Int64(int64(filer.FileSize(entry))), + ETag: aws.String(partETag), + } + output.Part = append(output.Part, part) + glog.V(3).Infof("listObjectParts: Added part %d, size=%d, etag=%s", + partNumber, filer.FileSize(entry), partETag) + if truncated { + output.NextPartNumberMarker = aws.Int64(int64(partNumber)) } } @@ -1672,6 +1661,82 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP return } +// listPartAttempts returns the latest stored attempt of each of the next +// max-parts part numbers after the marker. Each UploadPart writes its own +// entry, so a part number uploaded twice has several, and they may straddle a +// listing page. Names sort as text, which puts the one five-digit part number +// between "1000.part" and "1000_.part", so it is listed last, apart. +func (s3a *S3ApiServer) listPartAttempts(input *s3.ListPartsInput) (numbers []int, latest map[int]*filer_pb.Entry, truncated bool, err error) { + marker := *input.PartNumberMarker + maxParts := int(*input.MaxParts) + lastPart := s3_constants.MaxS3MultipartParts + latest = make(map[int]*filer_pb.Entry) + visit := func(partNumber int, entry *filer_pb.Entry) (full bool) { + prior, seen := latest[partNumber] + if !seen { + if maxParts > 0 && len(numbers) == maxParts { + return true + } + numbers = append(numbers, partNumber) + } + if !seen || compareLatestPartAttempt(entry, prior) < 0 { + latest[partNumber] = entry + } + return false + } + if marker+1 < int64(lastPart) { + // the marker is exclusive even if a store ignores startFrom + truncated, err = s3a.scanPartAttempts(input, "", fmt.Sprintf("%04d", marker+1), func(partNumber int) bool { + return int64(partNumber) > marker && partNumber < lastPart + }, visit) + if err != nil || truncated { + return numbers, latest, truncated, err + } + } + if marker >= int64(lastPart) { + return numbers, latest, false, nil + } + truncated, err = s3a.scanPartAttempts(input, strconv.Itoa(lastPart), strconv.Itoa(lastPart), func(partNumber int) bool { + return partNumber == lastPart + }, visit) + return numbers, latest, truncated, err +} + +// scanPartAttempts feeds the part entries named from startFrom on to visit, in +// name order, until visit reports the page full. +func (s3a *S3ApiServer) scanPartAttempts(input *s3.ListPartsInput, prefix, startFrom string, wanted func(int) bool, visit func(int, *filer_pb.Entry) bool) (full bool, err error) { + uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId + var batch uint32 + if *input.MaxParts > 0 { + batch = uint32(*input.MaxParts) + 1 + } + inclusive := true + for { + entries, isLast, listErr := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, prefix, startFrom, inclusive, batch) + if listErr != nil { + return false, listErr + } + priorStart := startFrom + for _, entry := range entries { + startFrom, inclusive = entry.Name, false + if entry.IsDirectory || !strings.HasSuffix(entry.Name, multipartExt) { + continue + } + partNumber, parseErr := parsePartNumber(entry.Name) + if parseErr != nil { + glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, parseErr) + continue + } + if wanted(partNumber) && visit(partNumber, entry) { + return true, nil + } + } + if isLast || batch == 0 || startFrom == priorStart { + return false, nil + } + } +} + // maxInt returns the maximum of two int values func maxInt(a, b int) int { if a > b { @@ -1852,17 +1917,29 @@ func (s3a *S3ApiServer) applyMultipartEncryptionConfig(entry *filer_pb.Entry, co } } +// sortEntriesByLatestChunk orders part attempts newest first: by the filer's +// commit stamp (unstamped entries predate it), then by the later of the first +// chunk's write time and the entry mtime, so a chunkless attempt still ranks. func sortEntriesByLatestChunk(entries []*filer_pb.Entry) { - slices.SortFunc(entries, func(a, b *filer_pb.Entry) int { - var aTs, bTs int64 - if len(a.Chunks) > 0 { - aTs = a.Chunks[0].ModifiedTsNs - } - if len(b.Chunks) > 0 { - bTs = b.Chunks[0].ModifiedTsNs - } - return cmp.Compare(bTs, aTs) - }) + slices.SortFunc(entries, compareLatestPartAttempt) +} + +func compareLatestPartAttempt(a, b *filer_pb.Entry) int { + if c := cmp.Compare(b.CommitTsNs, a.CommitTsNs); c != 0 { + return c + } + return cmp.Compare(partAttemptTsNs(b), partAttemptTsNs(a)) +} + +func partAttemptTsNs(entry *filer_pb.Entry) int64 { + var tsNs int64 + if len(entry.Chunks) > 0 { + tsNs = entry.Chunks[0].ModifiedTsNs + } + if entry.Attributes != nil { + tsNs = max(tsNs, entry.Attributes.Mtime*int64(time.Second)+int64(entry.Attributes.MtimeNs)) + } + return tsNs } func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPartNumbers []int) string { diff --git a/weed/s3api/s3api_list_parts_no_such_upload_test.go b/weed/s3api/s3api_list_parts_no_such_upload_test.go index c894571bb0..44c2b0565e 100644 --- a/weed/s3api/s3api_list_parts_no_such_upload_test.go +++ b/weed/s3api/s3api_list_parts_no_such_upload_test.go @@ -2,8 +2,10 @@ package s3api import ( "context" + "fmt" "math" "sort" + "strings" "testing" "github.com/aws/aws-sdk-go/aws" @@ -49,6 +51,9 @@ func (f *fakePartsFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream fi var sent uint32 for _, entry := range parts { name := entry.Name + if !strings.HasPrefix(name, req.Prefix) { + continue + } if req.StartFromFileName != "" { if name < req.StartFromFileName { continue @@ -278,3 +283,213 @@ func partNumbers(output *ListPartsResult) []int64 { } return nums } + +func partAttempt(name string, modifiedTsNs int64, etag string) *filer_pb.Entry { + return &filer_pb.Entry{ + Name: name, + Attributes: &filer_pb.FuseAttributes{FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,01", Size: 5, ModifiedTsNs: modifiedTsNs}}, + Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(etag)}, + } +} + +func partETags(output *ListPartsResult) []string { + etags := make([]string, len(output.Part)) + for i, part := range output.Part { + etags[i] = aws.StringValue(part.ETag) + } + return etags +} + +// Each UploadPart attempt keeps its own entry; ListParts reports one entry per +// part number, the latest attempt, the way a re-upload replaces a part on S3. +func TestListPartsReportsLatestAttemptPerPartNumber(t *testing.T) { + s3a := newListPartsServer(t, &fakePartsFiler{ + uploadEntry: uploadRecordEntry("open-upload"), + parts: []*filer_pb.Entry{ + partAttempt("0001_11111111-1111-1111-1111-111111111111.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + partAttempt("0002_22222222-2222-2222-2222-222222222222.part", 300, "cccccccccccccccccccccccccccccccc"), + }, + }) + + output, code := s3a.listObjectParts(listPartsInput("open-upload")) + if code != s3err.ErrNone { + t.Fatalf("code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 2 || got[0] != 1 || got[1] != 2 { + t.Fatalf("parts = %v, want [1 2]", got) + } + if got := partETags(output); got[0] != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" { + t.Fatalf("part 1 etag = %s, want the latest attempt", got[0]) + } + if aws.BoolValue(output.IsTruncated) { + t.Fatal("IsTruncated = true, want false") + } +} + +// With max-parts=1 the attempts of one part number span more than a page of +// raw entries; paging must still return each part number once, latest attempt +// first, and advance past the superseded entries. +func TestListPartsPagesOverRepeatedAttempts(t *testing.T) { + s3a := newListPartsServer(t, &fakePartsFiler{ + uploadEntry: uploadRecordEntry("open-upload"), + parts: []*filer_pb.Entry{ + partAttempt("0001_11111111-1111-1111-1111-111111111111.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + partAttempt("0001_55555555-5555-5555-5555-555555555555.part", 150, "dddddddddddddddddddddddddddddddd"), + partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + partAttempt("0002_22222222-2222-2222-2222-222222222222.part", 300, "cccccccccccccccccccccccccccccccc"), + }, + }) + + input := listPartsInput("open-upload") + input.MaxParts = aws.Int64(1) + + output, code := s3a.listObjectParts(input) + if code != s3err.ErrNone { + t.Fatalf("page 1 code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 1 || got[0] != 1 { + t.Fatalf("page 1 parts = %v, want [1]", got) + } + if got := partETags(output); got[0] != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" { + t.Fatalf("page 1 etag = %s, want the latest attempt", got[0]) + } + if !aws.BoolValue(output.IsTruncated) || aws.Int64Value(output.NextPartNumberMarker) != 1 { + t.Fatalf("page 1 IsTruncated/NextPartNumberMarker = %v/%v, want true/1", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker) + } + + input.PartNumberMarker = output.NextPartNumberMarker + output, code = s3a.listObjectParts(input) + if code != s3err.ErrNone { + t.Fatalf("page 2 code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 1 || got[0] != 2 { + t.Fatalf("page 2 parts = %v, want [2]", got) + } + if aws.BoolValue(output.IsTruncated) || output.NextPartNumberMarker != nil { + t.Fatalf("page 2 IsTruncated/NextPartNumberMarker = %v/%v, want false/nil", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker) + } +} + +// An empty re-upload has no chunks, so it has no chunk write time; it must +// still supersede the earlier non-empty attempt. +func TestListPartsEmptyReplacementSupersedesEarlierAttempt(t *testing.T) { + const emptyETag = "d41d8cd98f00b204e9800998ecf8427e" + for _, tc := range []struct { + name string + olderTs, newerTs int64 + olderSec, newerSec int64 + }{ + {name: "commit stamps", olderTs: 100, newerTs: 200}, + {name: "mtime without commit stamps", olderSec: 1, newerSec: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + older := partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 1_500_000_000, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + older.CommitTsNs = tc.olderTs + older.Attributes.Mtime = tc.olderSec + newer := &filer_pb.Entry{ + Name: "0001_11111111-1111-1111-1111-111111111111.part", + Attributes: &filer_pb.FuseAttributes{Mtime: tc.newerSec}, + Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(emptyETag)}, + CommitTsNs: tc.newerTs, + } + s3a := newListPartsServer(t, &fakePartsFiler{ + uploadEntry: uploadRecordEntry("open-upload"), + parts: []*filer_pb.Entry{older, newer}, + }) + + output, code := s3a.listObjectParts(listPartsInput("open-upload")) + if code != s3err.ErrNone { + t.Fatalf("code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 1 || got[0] != 1 { + t.Fatalf("parts = %v, want [1]", got) + } + if got := partETags(output)[0]; got != "\""+emptyETag+"\"" || aws.Int64Value(output.Part[0].Size) != 0 { + t.Fatalf("part 1 = %s/%d, want the empty replacement", got, aws.Int64Value(output.Part[0].Size)) + } + }) + } +} + +// Names sort as text, so part 10000 lands between part 1000's legacy and uuid +// names; every page size must still walk the part numbers in numeric order. +func TestListPartsPagesPartTenThousandLast(t *testing.T) { + s3a := newListPartsServer(t, &fakePartsFiler{ + uploadEntry: uploadRecordEntry("open-upload"), + parts: []*filer_pb.Entry{ + partAttempt("0999_99999999-9999-9999-9999-999999999999.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + partAttempt("1000_10001000-1000-1000-1000-100010001000.part", 100, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + partAttempt("1001_10011001-1001-1001-1001-100110011001.part", 100, "cccccccccccccccccccccccccccccccc"), + partAttempt("10000_10000000-0000-0000-0000-100000000000.part", 100, "dddddddddddddddddddddddddddddddd"), + }, + }) + + for maxParts := int64(1); maxParts <= 5; maxParts++ { + input := listPartsInput("open-upload") + input.MaxParts = aws.Int64(maxParts) + var got []int64 + for page := 0; page < 5; page++ { + output, code := s3a.listObjectParts(input) + if code != s3err.ErrNone { + t.Fatalf("max-parts %d page %d code = %v, want ErrNone", maxParts, page, code) + } + if int64(len(output.Part)) > maxParts { + t.Fatalf("max-parts %d page %d returned %d parts", maxParts, page, len(output.Part)) + } + got = append(got, partNumbers(output)...) + if !aws.BoolValue(output.IsTruncated) { + break + } + input.PartNumberMarker = output.NextPartNumberMarker + } + if fmt.Sprint(got) != "[999 1000 1001 10000]" { + t.Fatalf("max-parts %d parts = %v, want [999 1000 1001 10000]", maxParts, got) + } + } +} + +// Part 10000's names sort inside part 1000's: a legacy 1000.part, then +// 10000_.part, then the newer 1000_.part. Part 1000 must still be +// reported from its newer attempt. +func TestListPartsPartTenThousandDoesNotSplitPartAttempts(t *testing.T) { + s3a := newListPartsServer(t, &fakePartsFiler{ + uploadEntry: uploadRecordEntry("open-upload"), + parts: []*filer_pb.Entry{ + partAttempt("1000.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + partAttempt("10000_10000000-0000-0000-0000-100000000000.part", 150, "cccccccccccccccccccccccccccccccc"), + partAttempt("1000_10001000-1000-1000-1000-100010001000.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + }, + }) + + input := listPartsInput("open-upload") + input.MaxParts = aws.Int64(1) + input.PartNumberMarker = aws.Int64(999) + + output, code := s3a.listObjectParts(input) + if code != s3err.ErrNone { + t.Fatalf("page 1 code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 1 || got[0] != 1000 { + t.Fatalf("page 1 parts = %v, want [1000]", got) + } + if got := partETags(output)[0]; got != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" { + t.Fatalf("part 1000 etag = %s, want the newer attempt", got) + } + if !aws.BoolValue(output.IsTruncated) || aws.Int64Value(output.NextPartNumberMarker) != 1000 { + t.Fatalf("page 1 IsTruncated/NextPartNumberMarker = %v/%v, want true/1000", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker) + } + + input.PartNumberMarker = output.NextPartNumberMarker + output, code = s3a.listObjectParts(input) + if code != s3err.ErrNone { + t.Fatalf("page 2 code = %v, want ErrNone", code) + } + if got := partNumbers(output); len(got) != 1 || got[0] != 10000 { + t.Fatalf("page 2 parts = %v, want [10000]", got) + } + if aws.BoolValue(output.IsTruncated) { + t.Fatal("page 2 IsTruncated = true, want false") + } +}