s3: ListParts returns each part number once, its latest attempt (#12770)

* s3: list each part number once, its latest attempt

Every UploadPart writes its own NNNN_<uuid>.part entry, so a part number
uploaded twice listed twice, superseded ETag included, and a max-parts page
could hold only the stale attempt. ListParts now groups the entries by part
number, reports the attempt sortEntriesByLatestChunk ranks first (the same
order completion uses), and pages over distinct part numbers, reading past a
page boundary that splits one part number's attempts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: test ListParts over repeated part attempts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: rank an empty part attempt by commit stamp or mtime

An empty UploadPart body stores no chunks, so sortEntriesByLatestChunk gave
it time zero and an empty re-upload always lost to the attempt it replaced.
Attempts now rank by the filer's commit stamp when both carry one, else by
the later of the first chunk's write time and the entry mtime.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: keep only the newest attempt per part while listing parts

listPartAttempts held every attempt of a part number until the page was
built, so a heavily retried part grew one ListParts request's memory past
what max-parts bounds. It now compares each entry against the newest seen
for its part number with compareLatestPartAttempt, the comparator
sortEntriesByLatestChunk sorts with.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: order part attempts by commit stamp, then attempt time

Comparing commit stamps only when both attempts carried one made the order
non-transitive across stamped and unstamped attempts, so listing and
completion could pick different attempts depending on input order. The
commit stamp now leads (unstamped entries predate the field) and the
attempt time breaks ties.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: list part 10000 after the four-digit part numbers

Part names sort as text, so 10000_<uuid>.part lands between 1000.part and
1000_<uuid>.part. A page could end on part 10000 and the next marker then
skipped parts 1000-9999, and a legacy 1000.part could be reported before
the newer attempt of part 1000 was read. List the four-digit part numbers
first and part 10000 in its own prefix listing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3: test ListParts paging across part 10000

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-19 12:36:27 -08:00
1 parent dcbea0626f
commit 668ca45029
2 files changed
+329 -37

No files matched your search

+114 -37
View File
@@ -1618,7 +1618,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
output.IsTruncated = aws.Bool(false)
return output, s3err.ErrNone
}
entries, isLast, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts))
numbers, latest, truncated, err := s3a.listPartAttempts(input)
if err != nil {
// A store that reports the missing upload directory as not-found means
// the upload is gone (completed or aborted), not a store error.
@@ -1632,33 +1632,22 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
// Note: The upload directory is sort of a marker of the existence of an multipart upload request.
// So can not just delete empty upload folders.
output.IsTruncated = aws.Bool(!isLast)
output.IsTruncated = aws.Bool(truncated)
for _, entry := range entries {
if strings.HasSuffix(entry.Name, multipartExt) && !entry.IsDirectory {
partNumber, err := parsePartNumber(entry.Name)
if err != nil {
glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err)
continue
}
// the marker is exclusive even if a store ignores startFrom
if partNumber <= int(*input.PartNumberMarker) {
continue
}
partETag := getEtagFromEntry(entry)
part := &s3.Part{
PartNumber: aws.Int64(int64(partNumber)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()),
Size: aws.Int64(int64(filer.FileSize(entry))),
ETag: aws.String(partETag),
}
output.Part = append(output.Part, part)
glog.V(3).Infof("listObjectParts: Added part %d, size=%d, etag=%s",
partNumber, filer.FileSize(entry), partETag)
if !isLast {
output.NextPartNumberMarker = aws.Int64(int64(partNumber))
}
for _, partNumber := range numbers {
entry := latest[partNumber]
partETag := getEtagFromEntry(entry)
part := &s3.Part{
PartNumber: aws.Int64(int64(partNumber)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()),
Size: aws.Int64(int64(filer.FileSize(entry))),
ETag: aws.String(partETag),
}
output.Part = append(output.Part, part)
glog.V(3).Infof("listObjectParts: Added part %d, size=%d, etag=%s",
partNumber, filer.FileSize(entry), partETag)
if truncated {
output.NextPartNumberMarker = aws.Int64(int64(partNumber))
}
}
@@ -1672,6 +1661,82 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
return
}
// listPartAttempts returns the latest stored attempt of each of the next
// max-parts part numbers after the marker. Each UploadPart writes its own
// entry, so a part number uploaded twice has several, and they may straddle a
// listing page. Names sort as text, which puts the one five-digit part number
// between "1000.part" and "1000_<uuid>.part", so it is listed last, apart.
func (s3a *S3ApiServer) listPartAttempts(input *s3.ListPartsInput) (numbers []int, latest map[int]*filer_pb.Entry, truncated bool, err error) {
marker := *input.PartNumberMarker
maxParts := int(*input.MaxParts)
lastPart := s3_constants.MaxS3MultipartParts
latest = make(map[int]*filer_pb.Entry)
visit := func(partNumber int, entry *filer_pb.Entry) (full bool) {
prior, seen := latest[partNumber]
if !seen {
if maxParts > 0 && len(numbers) == maxParts {
return true
}
numbers = append(numbers, partNumber)
}
if !seen || compareLatestPartAttempt(entry, prior) < 0 {
latest[partNumber] = entry
}
return false
}
if marker+1 < int64(lastPart) {
// the marker is exclusive even if a store ignores startFrom
truncated, err = s3a.scanPartAttempts(input, "", fmt.Sprintf("%04d", marker+1), func(partNumber int) bool {
return int64(partNumber) > marker && partNumber < lastPart
}, visit)
if err != nil || truncated {
return numbers, latest, truncated, err
}
}
if marker >= int64(lastPart) {
return numbers, latest, false, nil
}
truncated, err = s3a.scanPartAttempts(input, strconv.Itoa(lastPart), strconv.Itoa(lastPart), func(partNumber int) bool {
return partNumber == lastPart
}, visit)
return numbers, latest, truncated, err
}
// scanPartAttempts feeds the part entries named from startFrom on to visit, in
// name order, until visit reports the page full.
func (s3a *S3ApiServer) scanPartAttempts(input *s3.ListPartsInput, prefix, startFrom string, wanted func(int) bool, visit func(int, *filer_pb.Entry) bool) (full bool, err error) {
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
var batch uint32
if *input.MaxParts > 0 {
batch = uint32(*input.MaxParts) + 1
}
inclusive := true
for {
entries, isLast, listErr := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, prefix, startFrom, inclusive, batch)
if listErr != nil {
return false, listErr
}
priorStart := startFrom
for _, entry := range entries {
startFrom, inclusive = entry.Name, false
if entry.IsDirectory || !strings.HasSuffix(entry.Name, multipartExt) {
continue
}
partNumber, parseErr := parsePartNumber(entry.Name)
if parseErr != nil {
glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, parseErr)
continue
}
if wanted(partNumber) && visit(partNumber, entry) {
return true, nil
}
}
if isLast || batch == 0 || startFrom == priorStart {
return false, nil
}
}
}
// maxInt returns the maximum of two int values
func maxInt(a, b int) int {
if a > b {
@@ -1852,17 +1917,29 @@ func (s3a *S3ApiServer) applyMultipartEncryptionConfig(entry *filer_pb.Entry, co
}
}
// sortEntriesByLatestChunk orders part attempts newest first: by the filer's
// commit stamp (unstamped entries predate it), then by the later of the first
// chunk's write time and the entry mtime, so a chunkless attempt still ranks.
func sortEntriesByLatestChunk(entries []*filer_pb.Entry) {
slices.SortFunc(entries, func(a, b *filer_pb.Entry) int {
var aTs, bTs int64
if len(a.Chunks) > 0 {
aTs = a.Chunks[0].ModifiedTsNs
}
if len(b.Chunks) > 0 {
bTs = b.Chunks[0].ModifiedTsNs
}
return cmp.Compare(bTs, aTs)
})
slices.SortFunc(entries, compareLatestPartAttempt)
}
func compareLatestPartAttempt(a, b *filer_pb.Entry) int {
if c := cmp.Compare(b.CommitTsNs, a.CommitTsNs); c != 0 {
return c
}
return cmp.Compare(partAttemptTsNs(b), partAttemptTsNs(a))
}
func partAttemptTsNs(entry *filer_pb.Entry) int64 {
var tsNs int64
if len(entry.Chunks) > 0 {
tsNs = entry.Chunks[0].ModifiedTsNs
}
if entry.Attributes != nil {
tsNs = max(tsNs, entry.Attributes.Mtime*int64(time.Second)+int64(entry.Attributes.MtimeNs))
}
return tsNs
}
func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPartNumbers []int) string {
@@ -2,8 +2,10 @@ package s3api
import (
"context"
"fmt"
"math"
"sort"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
@@ -49,6 +51,9 @@ func (f *fakePartsFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream fi
var sent uint32
for _, entry := range parts {
name := entry.Name
if !strings.HasPrefix(name, req.Prefix) {
continue
}
if req.StartFromFileName != "" {
if name < req.StartFromFileName {
continue
@@ -278,3 +283,213 @@ func partNumbers(output *ListPartsResult) []int64 {
}
return nums
}
func partAttempt(name string, modifiedTsNs int64, etag string) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{FileSize: 5},
Chunks: []*filer_pb.FileChunk{{FileId: "1,01", Size: 5, ModifiedTsNs: modifiedTsNs}},
Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(etag)},
}
}
func partETags(output *ListPartsResult) []string {
etags := make([]string, len(output.Part))
for i, part := range output.Part {
etags[i] = aws.StringValue(part.ETag)
}
return etags
}
// Each UploadPart attempt keeps its own entry; ListParts reports one entry per
// part number, the latest attempt, the way a re-upload replaces a part on S3.
func TestListPartsReportsLatestAttemptPerPartNumber(t *testing.T) {
s3a := newListPartsServer(t, &fakePartsFiler{
uploadEntry: uploadRecordEntry("open-upload"),
parts: []*filer_pb.Entry{
partAttempt("0001_11111111-1111-1111-1111-111111111111.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
partAttempt("0002_22222222-2222-2222-2222-222222222222.part", 300, "cccccccccccccccccccccccccccccccc"),
},
})
output, code := s3a.listObjectParts(listPartsInput("open-upload"))
if code != s3err.ErrNone {
t.Fatalf("code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 2 || got[0] != 1 || got[1] != 2 {
t.Fatalf("parts = %v, want [1 2]", got)
}
if got := partETags(output); got[0] != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" {
t.Fatalf("part 1 etag = %s, want the latest attempt", got[0])
}
if aws.BoolValue(output.IsTruncated) {
t.Fatal("IsTruncated = true, want false")
}
}
// With max-parts=1 the attempts of one part number span more than a page of
// raw entries; paging must still return each part number once, latest attempt
// first, and advance past the superseded entries.
func TestListPartsPagesOverRepeatedAttempts(t *testing.T) {
s3a := newListPartsServer(t, &fakePartsFiler{
uploadEntry: uploadRecordEntry("open-upload"),
parts: []*filer_pb.Entry{
partAttempt("0001_11111111-1111-1111-1111-111111111111.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
partAttempt("0001_55555555-5555-5555-5555-555555555555.part", 150, "dddddddddddddddddddddddddddddddd"),
partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
partAttempt("0002_22222222-2222-2222-2222-222222222222.part", 300, "cccccccccccccccccccccccccccccccc"),
},
})
input := listPartsInput("open-upload")
input.MaxParts = aws.Int64(1)
output, code := s3a.listObjectParts(input)
if code != s3err.ErrNone {
t.Fatalf("page 1 code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 1 || got[0] != 1 {
t.Fatalf("page 1 parts = %v, want [1]", got)
}
if got := partETags(output); got[0] != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" {
t.Fatalf("page 1 etag = %s, want the latest attempt", got[0])
}
if !aws.BoolValue(output.IsTruncated) || aws.Int64Value(output.NextPartNumberMarker) != 1 {
t.Fatalf("page 1 IsTruncated/NextPartNumberMarker = %v/%v, want true/1", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker)
}
input.PartNumberMarker = output.NextPartNumberMarker
output, code = s3a.listObjectParts(input)
if code != s3err.ErrNone {
t.Fatalf("page 2 code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 1 || got[0] != 2 {
t.Fatalf("page 2 parts = %v, want [2]", got)
}
if aws.BoolValue(output.IsTruncated) || output.NextPartNumberMarker != nil {
t.Fatalf("page 2 IsTruncated/NextPartNumberMarker = %v/%v, want false/nil", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker)
}
}
// An empty re-upload has no chunks, so it has no chunk write time; it must
// still supersede the earlier non-empty attempt.
func TestListPartsEmptyReplacementSupersedesEarlierAttempt(t *testing.T) {
const emptyETag = "d41d8cd98f00b204e9800998ecf8427e"
for _, tc := range []struct {
name string
olderTs, newerTs int64
olderSec, newerSec int64
}{
{name: "commit stamps", olderTs: 100, newerTs: 200},
{name: "mtime without commit stamps", olderSec: 1, newerSec: 2},
} {
t.Run(tc.name, func(t *testing.T) {
older := partAttempt("0001_99999999-9999-9999-9999-999999999999.part", 1_500_000_000, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
older.CommitTsNs = tc.olderTs
older.Attributes.Mtime = tc.olderSec
newer := &filer_pb.Entry{
Name: "0001_11111111-1111-1111-1111-111111111111.part",
Attributes: &filer_pb.FuseAttributes{Mtime: tc.newerSec},
Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(emptyETag)},
CommitTsNs: tc.newerTs,
}
s3a := newListPartsServer(t, &fakePartsFiler{
uploadEntry: uploadRecordEntry("open-upload"),
parts: []*filer_pb.Entry{older, newer},
})
output, code := s3a.listObjectParts(listPartsInput("open-upload"))
if code != s3err.ErrNone {
t.Fatalf("code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 1 || got[0] != 1 {
t.Fatalf("parts = %v, want [1]", got)
}
if got := partETags(output)[0]; got != "\""+emptyETag+"\"" || aws.Int64Value(output.Part[0].Size) != 0 {
t.Fatalf("part 1 = %s/%d, want the empty replacement", got, aws.Int64Value(output.Part[0].Size))
}
})
}
}
// Names sort as text, so part 10000 lands between part 1000's legacy and uuid
// names; every page size must still walk the part numbers in numeric order.
func TestListPartsPagesPartTenThousandLast(t *testing.T) {
s3a := newListPartsServer(t, &fakePartsFiler{
uploadEntry: uploadRecordEntry("open-upload"),
parts: []*filer_pb.Entry{
partAttempt("0999_99999999-9999-9999-9999-999999999999.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
partAttempt("1000_10001000-1000-1000-1000-100010001000.part", 100, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
partAttempt("1001_10011001-1001-1001-1001-100110011001.part", 100, "cccccccccccccccccccccccccccccccc"),
partAttempt("10000_10000000-0000-0000-0000-100000000000.part", 100, "dddddddddddddddddddddddddddddddd"),
},
})
for maxParts := int64(1); maxParts <= 5; maxParts++ {
input := listPartsInput("open-upload")
input.MaxParts = aws.Int64(maxParts)
var got []int64
for page := 0; page < 5; page++ {
output, code := s3a.listObjectParts(input)
if code != s3err.ErrNone {
t.Fatalf("max-parts %d page %d code = %v, want ErrNone", maxParts, page, code)
}
if int64(len(output.Part)) > maxParts {
t.Fatalf("max-parts %d page %d returned %d parts", maxParts, page, len(output.Part))
}
got = append(got, partNumbers(output)...)
if !aws.BoolValue(output.IsTruncated) {
break
}
input.PartNumberMarker = output.NextPartNumberMarker
}
if fmt.Sprint(got) != "[999 1000 1001 10000]" {
t.Fatalf("max-parts %d parts = %v, want [999 1000 1001 10000]", maxParts, got)
}
}
}
// Part 10000's names sort inside part 1000's: a legacy 1000.part, then
// 10000_<uuid>.part, then the newer 1000_<uuid>.part. Part 1000 must still be
// reported from its newer attempt.
func TestListPartsPartTenThousandDoesNotSplitPartAttempts(t *testing.T) {
s3a := newListPartsServer(t, &fakePartsFiler{
uploadEntry: uploadRecordEntry("open-upload"),
parts: []*filer_pb.Entry{
partAttempt("1000.part", 100, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
partAttempt("10000_10000000-0000-0000-0000-100000000000.part", 150, "cccccccccccccccccccccccccccccccc"),
partAttempt("1000_10001000-1000-1000-1000-100010001000.part", 200, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
},
})
input := listPartsInput("open-upload")
input.MaxParts = aws.Int64(1)
input.PartNumberMarker = aws.Int64(999)
output, code := s3a.listObjectParts(input)
if code != s3err.ErrNone {
t.Fatalf("page 1 code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 1 || got[0] != 1000 {
t.Fatalf("page 1 parts = %v, want [1000]", got)
}
if got := partETags(output)[0]; got != "\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"" {
t.Fatalf("part 1000 etag = %s, want the newer attempt", got)
}
if !aws.BoolValue(output.IsTruncated) || aws.Int64Value(output.NextPartNumberMarker) != 1000 {
t.Fatalf("page 1 IsTruncated/NextPartNumberMarker = %v/%v, want true/1000", aws.BoolValue(output.IsTruncated), output.NextPartNumberMarker)
}
input.PartNumberMarker = output.NextPartNumberMarker
output, code = s3a.listObjectParts(input)
if code != s3err.ErrNone {
t.Fatalf("page 2 code = %v, want ErrNone", code)
}
if got := partNumbers(output); len(got) != 1 || got[0] != 10000 {
t.Fatalf("page 2 parts = %v, want [10000]", got)
}
if aws.BoolValue(output.IsTruncated) {
t.Fatal("page 2 IsTruncated = true, want false")
}
}