seaweedfs/weed/mount/content_flush_recovery.go

package mount

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/protobuf/proto"
)

// contentPublicationError keeps the exact failed request, before any later
// subscription event can change the live handle. Unwrap preserves refusal and
// transport classification.
type contentPublicationError struct {
	cause   error
	request *filer_pb.CreateEntryRequest
}

func (e *contentPublicationError) Error() string { return e.cause.Error() }
func (e *contentPublicationError) Unwrap() error { return e.cause }

// retainFailedContent saves the refused request and its original CAS base for
// operator recovery to a separate path. It never retries against a fresh base.
// failed-metadata is outside the per-mount cache directories that shutdown and
// restart remove. Records are private, immutable and never automatically replayed
// or removed: chunk references may include cipher keys. They do not prevent a
// volume.fsck run from collecting unreferenced chunks; recover before that sweep.
func (wfs *WFS) retainFailedContent(fh *FileHandle, dir, name string, cause error) (string, error) {
	root, err := wfs.failedContentDir()
	if err != nil {
		return "", err
	}
	if err = os.MkdirAll(root, 0700); err != nil {
		return "", err
	}
	var request *filer_pb.CreateEntryRequest
	var publication *contentPublicationError
	if errors.As(cause, &publication) {
		// Snapshot the request that failed, not a handle a later metadata event may
		// already have changed after the publication lock was released.
		request = proto.Clone(publication.request).(*filer_pb.CreateEntryRequest)
	} else {
		var entry *filer_pb.Entry
		fh.UpdateEntry(func(e *filer_pb.Entry) { entry = proto.Clone(e).(*filer_pb.Entry) })
		entry.Name = name
		wfs.mapPbIdFromLocalToFiler(entry)
		request = &filer_pb.CreateEntryRequest{Directory: dir, Entry: entry, Condition: wfs.failedContentCondition(fh), Signatures: []int32{wfs.signature}, SkipCheckParentDirectory: true}
	}
	record := struct {
		Path            string
		Error           string
		ClientSignature int32
		Request         *filer_pb.CreateEntryRequest
	}{
		Path: string(util.FullPath(request.Directory).Child(request.Entry.Name)), Error: cause.Error(), ClientSignature: wfs.signature, Request: request,
	}
	raw, err := json.Marshal(record)
	if err != nil {
		return "", err
	}
	file, err := os.CreateTemp(root, "pending-*.tmp")
	if err != nil {
		return "", err
	}
	temporary := file.Name()
	path := temporary + ".json"
	saved := false
	defer func() {
		file.Close()
		if !saved {
			os.Remove(temporary)
			os.Remove(path)
		}
	}()
	if _, err = file.Write(append(raw, '\n')); err != nil {
		return "", err
	}
	if err = file.Sync(); err != nil {
		return "", err
	}
	if err = file.Close(); err != nil {
		return "", err
	}
	if err = os.Rename(temporary, path); err != nil {
		return "", err
	}
	if err = util.FsyncDir(root); err != nil {
		return "", err
	}
	// Persist creation of failed-metadata itself when this is its first record.
	if err = util.FsyncDir(filepath.Dir(root)); err != nil {
		return "", err
	}
	saved = true
	return path, nil
}

// failedContentDir is per user: the write cache root defaults to the shared
// temp directory, where another user's private directory would refuse every save.
func (wfs *WFS) failedContentDir() (string, error) {
	if wfs.option.CacheDirForWrite == "" {
		return "", fmt.Errorf("write cache root unavailable for failed metadata retention")
	}
	name := "failed-metadata"
	if uid := os.Getuid(); uid >= 0 {
		name = fmt.Sprintf("%s-%d", name, uid)
	}
	return filepath.Abs(filepath.Join(wfs.option.CacheDirForWrite, name))
}

// Used only when failure happened before a publication request existed.
func (wfs *WFS) failedContentCondition(fh *FileHandle) *filer_pb.WriteCondition {
	clause := &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS}
	if base := fh.baseEntry.Load(); base != nil {
		expected := proto.Clone(base).(*filer_pb.Entry)
		wfs.mapPbIdFromLocalToFiler(expected)
		clause = &filer_pb.WriteCondition_Clause{Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, ExpectedEntry: expected}
	}
	return &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{clause}}
}