mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-06 20:17:38 +01:00
Chris Lu
e610dad2a1
Implement the gRPC service surface of the pushdown daemon: - Service struct embeds the generated UnimplementedSeaweed ParquetPushdownServer, exposes Ping (returns version + trust mode) and Pushdown (returns Unimplemented after request-shape validation). - request_validation.go enforces hard caps (max data files, columns, predicate bytes, row-id cap, vector dim, top_k) and content-type rules (Puffin DV requires blob_length + referenced_data_file; equality delete requires equality_field_ids; predicate_kind and predicate must be set together). - stats.go is a small accumulator that produces a PushdownStats message stamped with trust_mode and elapsed micros; M0 wires it into the Unimplemented status's details so the stats wire format is exercised end to end. - TrustMode constants encode the design's catalog-validated (default) and connector-trusted (dev-only) modes. Unit tests cover the validation matrix (rejects empty/oversize requests, accepts well-formed vector queries, accepts well-formed deletion-vector descriptors, rejects DVs missing blob_length / referenced_data_file).
100 lines
3.3 KiB
Go
100 lines
3.3 KiB
Go
// Package parquet_pushdown implements the SeaweedFS-aware Parquet
|
|
// pushdown service. See PARQUET_PUSHDOWN_DESIGN.md and
|
|
// PARQUET_PUSHDOWN_DEV_PLAN.md at the repo root for the surrounding
|
|
// design and the milestone plan.
|
|
//
|
|
// M0 wires up the gRPC service and request validation; the actual
|
|
// pruning logic (parsed-footer cache, row-group pruning, scalar/page
|
|
// indexes, deletes, vectors) lands in M1+.
|
|
package parquet_pushdown
|
|
|
|
import (
|
|
"context"
|
|
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
|
|
pb "github.com/seaweedfs/seaweedfs/weed/pb/parquet_pushdown_pb"
|
|
)
|
|
|
|
// TrustMode controls how the server validates the request's
|
|
// DataFiles and Deletes against the Iceberg catalog. See the design's
|
|
// "Trust Model and Catalog Validation" section.
|
|
type TrustMode string
|
|
|
|
const (
|
|
// TrustModeCatalogValidated is the default. The server reads the
|
|
// Iceberg snapshot and verifies every DataFileDescriptor and
|
|
// DeleteFileRef against the manifest before serving.
|
|
TrustModeCatalogValidated TrustMode = "catalog-validated"
|
|
|
|
// TrustModeConnectorTrusted is a developer-only mode that skips
|
|
// catalog validation. Production builds must reject this mode at
|
|
// configuration time.
|
|
TrustModeConnectorTrusted TrustMode = "connector-trusted"
|
|
)
|
|
|
|
// Options configures a Service. Fields the M0 skeleton does not yet
|
|
// consume (catalog client, filer client, predicate engine) will be
|
|
// added in their respective milestones.
|
|
type Options struct {
|
|
// Version is reported in PingResponse and stamped on PushdownStats.
|
|
Version string
|
|
|
|
// TrustMode determines request-validation strictness. M0 only
|
|
// stamps the value into the response; the actual validation
|
|
// against the catalog lands in M3.
|
|
TrustMode TrustMode
|
|
}
|
|
|
|
// Service implements parquet_pushdown_pb.SeaweedParquetPushdownServer.
|
|
// One Service instance per daemon process.
|
|
type Service struct {
|
|
pb.UnimplementedSeaweedParquetPushdownServer
|
|
|
|
version string
|
|
trustMode TrustMode
|
|
}
|
|
|
|
// New constructs a Service from Options. The caller is responsible
|
|
// for registering it on a gRPC server (see weed/parquet_pushdown/daemon).
|
|
func New(opts Options) *Service {
|
|
mode := opts.TrustMode
|
|
if mode == "" {
|
|
mode = TrustModeCatalogValidated
|
|
}
|
|
return &Service{
|
|
version: opts.Version,
|
|
trustMode: mode,
|
|
}
|
|
}
|
|
|
|
// Ping returns daemon liveness information. Used by smoke tests and
|
|
// connector health checks. Cheap; never reads files.
|
|
func (s *Service) Ping(_ context.Context, _ *pb.PingRequest) (*pb.PingResponse, error) {
|
|
return &pb.PingResponse{
|
|
Version: s.version,
|
|
TrustMode: string(s.trustMode),
|
|
}, nil
|
|
}
|
|
|
|
// Pushdown evaluates a request and returns surviving file/row-group/
|
|
// page byte ranges. M0 returns Unimplemented after request-shape
|
|
// validation; later milestones replace the body.
|
|
//
|
|
// Unimplemented is attached with a PushdownStats detail so the wire
|
|
// format for stats is exercised end-to-end on M0.
|
|
func (s *Service) Pushdown(_ context.Context, req *pb.ParquetPushdownRequest) (*pb.ParquetPushdownResponse, error) {
|
|
stats := newStats(s.trustMode)
|
|
|
|
if err := validateRequest(req); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
st := status.New(codes.Unimplemented, "parquet pushdown evaluation lands in M1; M0 only validates request shape")
|
|
if withDetails, err := st.WithDetails(stats.toProto()); err == nil {
|
|
return nil, withDetails.Err()
|
|
}
|
|
return nil, st.Err()
|
|
}
|