mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 09:20:50 +01:00
Chris Lu
e1a09416fc
Wire the parser, cache, and Loader through the Service: - Options gains Loader and Cache (with CacheSize fallback). Defaults to LocalLoader and a footer-cache of defaultFooterCacheSize entries. - Pushdown handles the predicate-less, no-vector path by parsing each DataFile's footer (cached by Identity = (Path, SizeBytes, RecordCount, ETag)) and emitting one FileRange per (data file, row group, projected column) triple. Predicate / vector requests still return Unimplemented with stats. - statsRecorder gains per-request footer cache hit/miss counters that populate PushdownStats.footer_cache_hits / footer_cache_misses. - Column projection matches by ColumnRef.Path. Field-id-only refs return InvalidArgument until M3 (catalog-validated mode) plumbs the Iceberg schema through; the path-hint fallback is enough for the M1 round-trip test. Daemon supplies LocalLoader for now; the filer-backed Loader lands when M2+ needs it. Update the M0 Unimplemented test to send a predicate so it exercises the still-Unimplemented branch (file/range-level pushdown is now implemented for the no-predicate case).
46 lines
1.3 KiB
Go
46 lines
1.3 KiB
Go
package parquet_pushdown
|
|
|
|
import (
|
|
"time"
|
|
|
|
pb "github.com/seaweedfs/seaweedfs/weed/pb/parquet_pushdown_pb"
|
|
)
|
|
|
|
// statsRecorder accumulates per-request stats and converts to the
|
|
// wire form when the handler returns. Counts here are scoped to a
|
|
// single Pushdown call, not process-wide.
|
|
type statsRecorder struct {
|
|
start time.Time
|
|
trustMode TrustMode
|
|
footerCacheHits int64
|
|
footerCacheMisses int64
|
|
indexesUsed []string
|
|
indexesMissing []string
|
|
}
|
|
|
|
func newStats(trustMode TrustMode) *statsRecorder {
|
|
return &statsRecorder{start: time.Now(), trustMode: trustMode}
|
|
}
|
|
|
|
func (r *statsRecorder) recordFooterCacheHit() { r.footerCacheHits++ }
|
|
func (r *statsRecorder) recordFooterCacheMiss() { r.footerCacheMisses++ }
|
|
|
|
func (r *statsRecorder) markIndexUsed(kind string) {
|
|
r.indexesUsed = append(r.indexesUsed, kind)
|
|
}
|
|
|
|
func (r *statsRecorder) markIndexMissing(kind string) {
|
|
r.indexesMissing = append(r.indexesMissing, kind)
|
|
}
|
|
|
|
func (r *statsRecorder) toProto() *pb.PushdownStats {
|
|
return &pb.PushdownStats{
|
|
TrustMode: string(r.trustMode),
|
|
ServerTimeMicros: time.Since(r.start).Microseconds(),
|
|
FooterCacheHits: r.footerCacheHits,
|
|
FooterCacheMisses: r.footerCacheMisses,
|
|
IndexesUsed: r.indexesUsed,
|
|
IndexesMissing: r.indexesMissing,
|
|
}
|
|
}
|