Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-17 03:50:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

s3: add retry logic for filer consistency in versioned object operations

Browse Source 
- Add retry logic to updateLatestVersionInDirectory to handle cases where
  .versions directory creation succeeds but is not immediately visible
- Add retry logic to getLatestObjectVersion for the same consistency issue
- Use 3 retries with 50ms delays to handle filer store consistency timing
- Addresses CI failures where 'filer: no entry is found in filer store'
  occurs after successful directory creation
- Maintains CI debug logging to track retry attempts and outcomes
This commit is contained in:
chrislu
2025-09-14 22:24:53 -07:00
parent 1f275afc8f
commit 581392638b
2 changed files with 41 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/s3api/s3api_object_handlers_put.go
+21 -5
View File
@@ -640,12 +640,28 @@ func (s3a *S3ApiServer) updateLatestVersionInDirectory(bucket, object, versionId
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: starting update for %s/%s version %s file %s", bucket, object, versionId, versionFileName)
// Get the current .versions directory entry
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
// Get the current .versions directory entry with retry logic for filer consistency
var versionsEntry *filer_pb.Entry
var err error
maxRetries := 3
for attempt := 1; attempt <= maxRetries; attempt++ {
versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil {
break
}
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: attempt %d/%d failed to get .versions entry for %s/%s: %v", attempt, maxRetries, bucket, object, err)
if attempt < maxRetries {
// Brief wait before retry to allow filer consistency
time.Sleep(time.Millisecond * 50)
}
}
if err != nil {
glog.Errorf("updateLatestVersionInDirectory: failed to get .versions directory for %s/%s: %v", bucket, object, err)
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: FAILED to get .versions entry for %s/%s: %v", bucket, object, err)
return fmt.Errorf("failed to get .versions directory: %w", err)
glog.Errorf("updateLatestVersionInDirectory: failed to get .versions directory for %s/%s after %d attempts: %v", bucket, object, maxRetries, err)
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: FAILED to get .versions entry for %s/%s after %d attempts: %v", bucket, object, maxRetries, err)
return fmt.Errorf("failed to get .versions directory after %d attempts: %w", maxRetries, err)
}
glog.V(0).Infof("CI-DEBUG: updateLatestVersionInDirectory: got .versions entry for %s/%s, updating metadata", bucket, object)
weed/s3api/s3api_object_versioning.go
+20 -4
View File
@@ -782,14 +782,30 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
glog.V(1).Infof("getLatestObjectVersion: looking for latest version of %s/%s", bucket, object)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: starting lookup for %s/%s", bucket, object)
// Get the .versions directory entry to read latest version metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
// Get the .versions directory entry to read latest version metadata with retry logic for filer consistency
var versionsEntry *filer_pb.Entry
var err error
maxRetries := 3
for attempt := 1; attempt <= maxRetries; attempt++ {
versionsEntry, err = s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil {
break
}
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: attempt %d/%d failed to get .versions directory for %s/%s: %v", attempt, maxRetries, bucket, object, err)
if attempt < maxRetries {
// Brief wait before retry to allow filer consistency
time.Sleep(time.Millisecond * 50)
}
}
if err != nil {
// .versions directory doesn't exist - this can happen for objects that existed
// before versioning was enabled on the bucket. Fall back to checking for a
// regular (non-versioned) object file.
glog.V(1).Infof("getLatestObjectVersion: no .versions directory for %s%s (error: %v), checking for pre-versioning object", bucket, object, err)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: no .versions directory for %s/%s (error: %v), falling back to pre-versioning", bucket, object, err)
glog.V(1).Infof("getLatestObjectVersion: no .versions directory for %s%s after %d attempts (error: %v), checking for pre-versioning object", bucket, object, maxRetries, err)
glog.V(0).Infof("CI-DEBUG: getLatestObjectVersion: no .versions directory for %s/%s after %d attempts (error: %v), falling back to pre-versioning", bucket, object, maxRetries, err)
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
if regularErr != nil {