diff --git a/seaweed-vector/Cargo.lock b/seaweed-vector/Cargo.lock index 8b9da51181..e0b6355053 100644 --- a/seaweed-vector/Cargo.lock +++ b/seaweed-vector/Cargo.lock @@ -6486,6 +6486,7 @@ dependencies = [ "clap", "futures", "lance", + "lance-arrow", "lance-linalg", "prost", "prost-types", diff --git a/seaweed-vector/crates/server/Cargo.toml b/seaweed-vector/crates/server/Cargo.toml index 7a8018aee1..71f7f449a2 100644 --- a/seaweed-vector/crates/server/Cargo.toml +++ b/seaweed-vector/crates/server/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" weed-vector-core = { path = "../core" } weed-lance-worker = { path = "../../../seaweed-worker/crates/lance" } lance = { version = "10", default-features = false, features = ["aws"] } +lance-arrow = "10" lance-linalg = "10" arrow-array = "58" arrow-cast = "58" diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index e49a357ff2..e39882e0c8 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -110,8 +110,18 @@ fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc) -> Re .flat_map(|v| v.data.iter().copied()) .collect::>(); let values: ArrayRef = Arc::new(Float32Array::from(flat)); + // Lance stores the list item as nullable even when the write schema + // declared it non-null, so the child field must come from the batch's + // schema rather than a fixed declaration. + let DataType::FixedSizeList(item_field, _) = schema + .field_with_name(VECTOR_COLUMN) + .map_err(anyhow::Error::from)? + .data_type() + else { + bail!("vector column is not a fixed-size list"); + }; let vectors_array = FixedSizeListArray::try_new( - Field::new("item", DataType::Float32, false).into(), + item_field.clone(), dimension as i32, values, None, diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 3f0e8ef0b5..583f51cb90 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -788,7 +788,7 @@ func filterExpr(filter map[string]interface{}) (string, error) { } func fieldClause(field string, cond interface{}) (string, error) { - col := fmt.Sprintf("json_get_str(`metadata`, '%s')", strings.ReplaceAll(field, "'", "''")) + col := fmt.Sprintf("json_get_string(`metadata`, '%s')", strings.ReplaceAll(field, "'", "''")) switch c := cond.(type) { case map[string]interface{}: parts := []string{}