s3: add retry logic for filer consistency in versioned object operations

- Add retry logic to updateLatestVersionInDirectory to handle cases where
  .versions directory creation succeeds but is not immediately visible
- Add retry logic to getLatestObjectVersion for the same consistency issue
- Use 3 retries with 50ms delays to handle filer store consistency timing
- Addresses CI failures where 'filer: no entry is found in filer store'
  occurs after successful directory creation
- Maintains CI debug logging to track retry attempts and outcomes
This commit is contained in:
chrislu committed 2025-09-14 22:24:53 -07:00
1 parent 1f275afc8f
commit 581392638b
2 files changed
+41 -9

No files matched your search

+20 -4
View File
@@ -782,14 +782,30 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
glog.V(1).Infof("getLatestObjectVersion: looking for latest version of %s/%s", bucket, object)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: starting lookup for %s/%s", bucket, object)
// Get the .versions directory entry to read latest version metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
// Get the .versions directory entry to read latest version metadata with retry logic for filer consistency
var versionsEntry *filer_pb.Entry
var err error
maxRetries := 3
for attempt := 1; attempt <= maxRetries; attempt++ {
versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil {
break
}
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: attempt %d/%d failed to get .versions directory for %s/%s: %v", attempt, maxRetries, bucket, object, err)
if attempt < maxRetries {
// Brief wait before retry to allow filer consistency
time.Sleep(time.Millisecond * 50)
}
}
if err != nil {
// .versions directory doesn't exist - this can happen for objects that existed
// before versioning was enabled on the bucket. Fall back to checking for a
// regular (non-versioned) object file.
glog.V(1).Infof("getLatestObjectVersion: no .versions directory for %s%s (error: %v), checking for pre-versioning object", bucket, object, err)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: no .versions directory for %s/%s (error: %v), falling back to pre-versioning", bucket, object, err)
glog.V(1).Infof("getLatestObjectVersion: no .versions directory for %s%s after %d attempts (error: %v), checking for pre-versioning object", bucket, object, maxRetries, err)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: no .versions directory for %s/%s after %d attempts (error: %v), falling back to pre-versioning", bucket, object, maxRetries, err)
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
if regularErr != nil {