mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-25 14:30:43 +01:00
Javier de la TorreandJavier de la Torre
763e2a2090
* iceberg: record geometry bounds on the data files compaction writes Compaction collected column statistics for every primitive column except geometry: the bound converter has no geometry case, so a compacted file's manifest entry carried value and null counts for a geometry column but no lower or upper bound. Readers that skip files by these bounds for a spatial filter (for example DuckDB for `geom && <box>`) had to open every compacted file, where the files before compaction could be skipped. The bound follows the spec's single-value serialization for geometry: the lower bound is the point (xmin, ymin), the upper bound (xmax, ymax), each an 8-byte little-endian double; z and m are not written. It is the union of the written file's row-group geospatial statistics, which parquet-go computes from every non-null value. A bound that is too small makes a reader skip rows, so any doubt gives none: a row group with non-null values but no geospatial statistics (parquet-go omits them when a value does not parse as WKB), a box that is not finite, an inverted box, or a chunk whose statistics cannot be read. A row group of nulls only constrains nothing. Geography gets no bound, as iceberg-go writes none for it (geodesic edges, boxes across the antimeridian). Tests: points and polygons far apart through a bin-packed compaction (the bound equals the union of every geometry), a sorted compaction (any bound covers every geometry), a column of nulls only, an unparseable value in a second row group (no bound; without the rule the file got the bound of the first row group only), geography, and the aggregation rules. * iceberg: do not order compaction bins by a geometry column When a table declares no usable sort order, compaction orders a bin's files by the first column every file has bounds for. A v3 table whose first column is a geometry qualifies as soon as its files carry geometry bounds (DuckDB writes them, and compacted files now do too). Those bounds decode into geo literals that compareLiterals cannot compare, so the order sorted nothing, yet an oversized bin was still split into "ordered" runs by splitOrderedBin. Geometry and geography columns are now skipped in both the declared sort order and the inferred one, so such a table is ordered by its next column with bounds. Test: TestResolveCompactionOrderSkipsGeometry, a geometry column first and an id column second; on master the order is the geometry column. --------- Co-authored-by: Javier de la Torre <jatorre@lakehousebox.com>
72 lines
2.3 KiB
Go
72 lines
2.3 KiB
Go
package iceberg
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"math"
|
|
|
|
"github.com/parquet-go/parquet-go/format"
|
|
)
|
|
|
|
// geometryBounds aggregates the bound of a geometry column over the row groups
|
|
// of a written file. The spec serializes a geometry bound as a point: the
|
|
// lower bound is (xmin, ymin), the upper bound (xmax, ymax), each coordinate
|
|
// an 8-byte little-endian double; z and m are optional and not written here.
|
|
//
|
|
// Readers skip a file whose bound does not intersect a spatial filter, so a
|
|
// box smaller than the file's geometries returns wrong results. Any doubt
|
|
// gives no bound:
|
|
// - the box is the union of the row groups' geospatial statistics, which
|
|
// the Parquet writer computes from every non-null value;
|
|
// - a row group with non-null values but no statistics (the writer omits
|
|
// them when a value does not parse as WKB) gives the column no bound, as
|
|
// does a box that is not finite or is inverted;
|
|
// - a row group of nulls only constrains nothing.
|
|
type geometryBounds struct {
|
|
xmin, ymin, xmax, ymax float64
|
|
seen bool
|
|
invalid bool
|
|
}
|
|
|
|
func (g *geometryBounds) add(gs format.GeospatialStatistics, numValues, nullCount int64) {
|
|
if g.invalid || numValues <= 0 || nullCount >= numValues {
|
|
return
|
|
}
|
|
if len(gs.GeoSpatialTypes) == 0 {
|
|
g.invalid = true
|
|
return
|
|
}
|
|
bb := gs.BBox
|
|
for _, v := range []float64{bb.XMin, bb.XMax, bb.YMin, bb.YMax} {
|
|
if math.IsNaN(v) || math.IsInf(v, 0) {
|
|
g.invalid = true
|
|
return
|
|
}
|
|
}
|
|
if bb.XMin > bb.XMax || bb.YMin > bb.YMax {
|
|
g.invalid = true
|
|
return
|
|
}
|
|
if !g.seen {
|
|
g.xmin, g.ymin, g.xmax, g.ymax, g.seen = bb.XMin, bb.YMin, bb.XMax, bb.YMax, true
|
|
return
|
|
}
|
|
g.xmin, g.ymin = math.Min(g.xmin, bb.XMin), math.Min(g.ymin, bb.YMin)
|
|
g.xmax, g.ymax = math.Max(g.xmax, bb.XMax), math.Max(g.ymax, bb.YMax)
|
|
}
|
|
|
|
// encoded returns the lower and upper bound, or false when the column gets
|
|
// none.
|
|
func (g *geometryBounds) encoded() (lower, upper []byte, ok bool) {
|
|
if g == nil || g.invalid || !g.seen {
|
|
return nil, nil, false
|
|
}
|
|
return encodeGeometryPoint(g.xmin, g.ymin), encodeGeometryPoint(g.xmax, g.ymax), true
|
|
}
|
|
|
|
func encodeGeometryPoint(x, y float64) []byte {
|
|
b := make([]byte, 16)
|
|
binary.LittleEndian.PutUint64(b, math.Float64bits(x))
|
|
binary.LittleEndian.PutUint64(b[8:], math.Float64bits(y))
|
|
return b
|
|
}
|