mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
filer,pb,mount: keep per-chunk cipher keys out of logs (#12085)
* filer: format ChunkView without its cipher key readChunkSliceAt logs a failed chunk fetch with %+v, which printed the per-chunk AES-256-GCM key into error-level logs on every failed read. Give ChunkView a String() that omits CipherKey so %v and %+v stay safe. * pb: scrub cipher keys when an entry must be logged whole ScrubCipherKeys clones an Entry with chunk CipherKey fields cleared, so a warning that needs the full entry still cannot leak the keys. * log entry identifiers instead of whole metadata messages Several log lines formatted whole protobuf messages — SubscribeMetadataResponse, EventNotification, Entry-carrying requests — whose chunks include cipher_key under -encryptVolumeData, so the keys escaped into logs (some at default verbosity). Log the directory, entry name and event timestamp instead; where a chunk list was printed, print file ids via FileIdsString. * log queue: unwrap SubscribeMetadataResponse and never print raw protos The fallback printed any message with %+v, so a SubscribeMetadataResponse carrying entries would still put cipher keys in the log. Unwrap its event notifications (batched events included) and reduce unknown messages to their type name. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * remote sync logs: name the event's real path Rename targets logged under resp.Directory while the entry lives under NewParentPath; route them through MetadataEventTargetDirectory. Sync failure logs printed only the directory, so same-dir failures shared an identifier — log the source and destination paths instead. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * log queue: log the entry's destination path on moves A cross-directory rename printed "file" -> "file" under the source path, hiding where the entry landed. Use NewParentPath so the log shows the destination directory next to the new name. * admin: install the go.mod-pinned templ, not @latest install-templ fetched templ@latest, which now emits templruntime.NewAttributeExpression - a symbol absent from the runtime pinned in go.mod - so every fresh 'make' regenerated uncompilable _templ.go files. Install the go.mod version and reinstall when the binary on PATH reports a different one. * admin: drop stray Makefile change belonging to the templ pin PR Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: join the move destination path in log queue output A NewParentPath of "/" printed //name for moves into the root; path.Join keeps a single leading slash. * log queue: skip a nil event notification instead of logging an empty move Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * pb: tolerate nil chunk entries while scrubbing cipher keys for logs Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * subscription and backup failure logs name the event's file The process/marshal failure logs printed only the event directory and timestamp, so a failed sync left operators guessing which file failed. Use MetadataEventSourceFullPath/MetadataEventTargetFullPath so each failure names the source and target paths without printing the entries. * replication: identify entry-less events without printing the proto Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: scrub cipher keys from the bucket-metadata debug dump buildBucketMetadata marshals the whole filer entry to JSON for a V(3) log line; a chunked entry would emit each chunk's cipher_key base64. Marshal the scrubbed copy instead. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
c27d7fe5a4
commit
3abf8b2572
26 files changed
+145
-44
No files matched your search
@@ -260,9 +260,9 @@ func processCreateEvent(
|
||||
if !filer.HasData(message.NewEntry) && !(mustSend && !isDroppedLocalCopy(message.NewEntry)) {
|
||||
return nil
|
||||
}
|
||||
glog.V(2).Infof("create: %+v", resp)
|
||||
glog.V(2).Infof("create: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
if !mustSend && !shouldSendToRemote(message.NewEntry) {
|
||||
glog.V(2).Infof("skipping creating: %+v", resp)
|
||||
glog.V(2).Infof("skipping creating: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
return nil
|
||||
}
|
||||
// Rewrite internal versioning paths to the original S3 key
|
||||
@@ -332,7 +332,7 @@ func processDeleteEvent(
|
||||
glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
|
||||
return nil
|
||||
}
|
||||
glog.V(2).Infof("delete: %+v", resp)
|
||||
glog.V(2).Infof("delete: %s/%s ts_ns=%d", resp.Directory, message.GetOldEntry().GetName(), resp.TsNs)
|
||||
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
|
||||
if !message.OldEntry.IsDirectory {
|
||||
// a deleted entry whose rename copy was still pending read its
|
||||
@@ -456,7 +456,7 @@ func processUpdateEvent(
|
||||
return fallback
|
||||
}
|
||||
if proto.Equal(oldDest, dest) && !shouldSendToRemote(message.NewEntry) {
|
||||
glog.V(2).Infof("skipping updating: %+v", resp)
|
||||
glog.V(2).Infof("skipping updating: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
return nil
|
||||
}
|
||||
if message.NewEntry.IsDirectory {
|
||||
@@ -465,14 +465,14 @@ func processUpdateEvent(
|
||||
if isMetadataOnlyUpdate(resp.Directory, message) {
|
||||
remoteEntry, err := liveRemoteEntry(filerClient, message.NewParentPath, message.NewEntry)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
glog.V(2).Infof("skipping updating deleted entry: %+v", resp)
|
||||
glog.V(2).Infof("skipping updating deleted entry: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if remoteEntry != nil {
|
||||
glog.V(2).Infof("update meta: %+v", resp)
|
||||
glog.V(2).Infof("update meta: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
updated, err := client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -494,10 +494,10 @@ func processUpdateEvent(
|
||||
// the entry as dirty when the remote mtime shares its second. The
|
||||
// content is the object at this very key, and uploading the chunkless
|
||||
// entry would replace it with zero bytes.
|
||||
glog.V(2).Infof("skipping update of remote-only entry: %+v", resp)
|
||||
glog.V(2).Infof("skipping update of remote-only entry: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
return nil
|
||||
}
|
||||
glog.V(2).Infof("update: %+v", resp)
|
||||
glog.V(2).Infof("update: %s/%s ts_ns=%d", filer_pb.MetadataEventTargetDirectory(resp), message.GetNewEntry().GetName(), resp.TsNs)
|
||||
if !proto.Equal(oldDest, dest) {
|
||||
// A renamed entry that holds no local data now (the snapshot was
|
||||
// remote-only, or remote.uncache ran since) has its content only as a
|
||||
|
||||
Reference in new issue
Block a user