mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
seaweed-vector: build a vector index from the CLI and measure it in the perf suite (ANN phase 0) (#12411)
* seaweed-vector: build and extend the vector index weed_lance_worker::vector_index builds an IVF_FLAT index in the metric the table was created with, sizes partitions from the row count, and extends an index with a new segment or a retrain; the core crate re-exports it and adds the probes a query spends. Tests cover coverage after build, append, retrain and compaction. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * weed-vector-cli: index build and index optimize Logs go to stderr so stdout stays the JSON a caller parses. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test/s3vectors: PERF_INDEX builds the index before the timed queries The cluster fixture hands out its catalog URL, the Makefile builds weed-vector-cli, and the perf suite builds the index after ingest and reports the build time and segment count. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: test that a build takes the table's metric Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test/s3vectors: make clean removes weed-vector-cli Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: optimize only the indices on the vector column Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: resolve the query metric with index::distance_metric Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-vector: test the L2 metric under the name CreateIndex stores Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: PERF_CLUSTERS draws the perf suite's vectors around shared centres Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * seaweed-vector: test that compaction drops the deleted key from tied hits Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-vector: test that an append-only optimize serves the new row Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: record PERF_NOISE in the perf report config Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: report recall through an index without asserting it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: count a hit tying the k-th best score toward recall Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: allow float32 rounding at the recall cutoff Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: count a repeated hit key once toward recall Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: credit cutoff ties only up to the slots the top-k leaves Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test/s3vectors: assert every filtered hit passes the filter, indexed or not Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
8350f297af
commit
f0f0ba85ae
13 files changed
+575
-34
No files matched your search
@@ -12,13 +12,13 @@ use lance::io::ObjectStoreParams;
|
||||
use lance::io::StorageOptionsAccessor;
|
||||
use lance::io::WrappingObjectStore;
|
||||
use lance::table::format::Fragment;
|
||||
use lance_linalg::distance::DistanceType;
|
||||
use prost_types::{ListValue, Struct, Value, value::Kind};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
use tonic::{Request, Response, Status};
|
||||
use weed_lance_worker::catalog::NamespaceClient;
|
||||
use weed_lance_worker::dataset::{self, FallbackOptions};
|
||||
use weed_vector_core::index;
|
||||
use weed_vector_core::query::{self, Query};
|
||||
use weed_vector_core::table::VectorTable;
|
||||
|
||||
@@ -658,16 +658,7 @@ impl VectorService for VectorServer {
|
||||
if req.top_k == 0 {
|
||||
return Err(Status::invalid_argument("top_k must be positive"));
|
||||
}
|
||||
// The index's configured metric lives in the vector field's metadata.
|
||||
let distance = match dataset
|
||||
.schema()
|
||||
.field(VECTOR_COLUMN)
|
||||
.and_then(|f| f.metadata.get("distance_metric").map(String::as_str))
|
||||
{
|
||||
Some("COSINE") => DistanceType::Cosine,
|
||||
Some("DOT_PRODUCT") => DistanceType::Dot,
|
||||
_ => DistanceType::L2,
|
||||
};
|
||||
let distance = index::distance_metric(&dataset, VECTOR_COLUMN);
|
||||
let query = Query {
|
||||
vector: req.query_vector,
|
||||
k: req.top_k as usize,
|
||||
|
||||
Reference in new issue
Block a user