mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 19:33:41 +01:00
Compare commits
71
Commits
No files matched your search
Generated
+3
@@ -7146,6 +7146,7 @@ dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -7192,6 +7193,7 @@ dependencies = [
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"x509-parser",
|
||||
"zstd",
|
||||
]
|
||||
@@ -7210,6 +7212,7 @@ dependencies = [
|
||||
"lance",
|
||||
"lance-index",
|
||||
"lance-linalg",
|
||||
"lance-select",
|
||||
"lance-table",
|
||||
"object_store",
|
||||
"roaring",
|
||||
|
||||
@@ -22,6 +22,7 @@ object_store = "0.13"
|
||||
lance-index = "10"
|
||||
lance-linalg = "10"
|
||||
lance-table = "10"
|
||||
lance-select = "10"
|
||||
arrow-array = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-schema = "58"
|
||||
|
||||
@@ -21,16 +21,16 @@ use lance::deps::datafusion::logical_expr::{
|
||||
};
|
||||
|
||||
/// Set `filter` on the scanner with the JSONPath functions swapped and the
|
||||
/// conjuncts on `ready` indexed keys answered by their columns. Returns the
|
||||
/// conjunction of those.
|
||||
/// conjuncts on `ready` indexed keys answered by their columns, which it
|
||||
/// returns.
|
||||
pub(crate) fn apply(
|
||||
scanner: &mut Scanner,
|
||||
filter: &str,
|
||||
ready: &HashMap<String, String>,
|
||||
) -> anyhow::Result<Option<Expr>> {
|
||||
) -> anyhow::Result<Vec<crate::indexed_filter::Indexed>> {
|
||||
scanner.filter(filter)?;
|
||||
let Some(expr) = scanner.get_expr_filter()? else {
|
||||
return Ok(None);
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let (expr, indexed) = crate::indexed_filter::rewrite(expr, ready);
|
||||
scanner.filter_expr(swap(expr)?);
|
||||
|
||||
@@ -32,19 +32,29 @@ pub(crate) fn ready_columns(dataset: &Dataset) -> HashMap<String, String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A conjunct a key's column answers: rows whose `column` holds one of
|
||||
/// `values`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct Indexed {
|
||||
pub column: String,
|
||||
pub values: Vec<String>,
|
||||
pub expr: Expr,
|
||||
}
|
||||
|
||||
/// `filter` with each top-level conjunct that is a string `$eq` or `$in` on
|
||||
/// a key in `ready` answered by the key's column instead, and the
|
||||
/// conjunction of those replacements.
|
||||
pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, Option<Expr>) {
|
||||
/// a key in `ready` answered by the key's column instead, and those
|
||||
/// replacements.
|
||||
pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, Vec<Indexed>) {
|
||||
if ready.is_empty() {
|
||||
return (filter, None);
|
||||
return (filter, Vec::new());
|
||||
}
|
||||
let mut indexed: Option<Expr> = None;
|
||||
let mut indexed: Vec<Indexed> = Vec::new();
|
||||
let mut rewritten: Option<Expr> = None;
|
||||
for conjunct in split_conjunction_owned(filter) {
|
||||
let replacement = string_membership(&conjunct).and_then(|(key, values)| {
|
||||
let column = Expr::Column(Column::from_name(ready.get(&key)?));
|
||||
Some(match values.as_slice() {
|
||||
let name = ready.get(&key)?;
|
||||
let column = Expr::Column(Column::from_name(name));
|
||||
let expr = match values.as_slice() {
|
||||
[one] => array_has(column, lit(one.as_str())),
|
||||
_ => {
|
||||
let values: Vec<ScalarValue> = values
|
||||
@@ -54,11 +64,17 @@ pub(crate) fn rewrite(filter: Expr, ready: &HashMap<String, String>) -> (Expr, O
|
||||
let list = ScalarValue::new_list(&values, &arrow_schema::DataType::Utf8, true);
|
||||
array_has_any(column, Expr::Literal(ScalarValue::List(list), None))
|
||||
}
|
||||
};
|
||||
Some(Indexed {
|
||||
column: name.clone(),
|
||||
values,
|
||||
expr,
|
||||
})
|
||||
});
|
||||
let part = match replacement {
|
||||
Some(expr) => {
|
||||
indexed = Some(and(indexed, expr.clone()));
|
||||
Some(found) => {
|
||||
let expr = found.expr.clone();
|
||||
indexed.push(found);
|
||||
expr
|
||||
}
|
||||
None => conjunct,
|
||||
@@ -221,6 +237,86 @@ fn jsonb_text(s: &str) -> Option<String> {
|
||||
Some(value.to_string())
|
||||
}
|
||||
|
||||
/// Segments searched at once when counting.
|
||||
const PARALLEL_SEGMENTS: usize = 8;
|
||||
|
||||
/// The rows of `scope` (every fragment when None) whose column matches
|
||||
/// `indexed`, or None once there are more than `limit`. Segments are
|
||||
/// searched newest first, several at once, and counting stops as soon as
|
||||
/// the total passes the limit. Rows the index has not reached are counted
|
||||
/// by a read of the column. Deleted rows an index segment still lists
|
||||
/// count too: the total is an upper bound, which only ever sends a query
|
||||
/// to the vector index instead of an exact scan.
|
||||
pub(crate) async fn count_indexed(
|
||||
dataset: &Dataset,
|
||||
indexed: &Indexed,
|
||||
scope: Option<&[u64]>,
|
||||
limit: u64,
|
||||
) -> Result<Option<u64>> {
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use lance::index::DatasetIndexInternalExt;
|
||||
use lance_index::metrics::NoOpMetricsCollector;
|
||||
use lance_index::scalar::{LabelListQuery, SearchResult};
|
||||
use lance_select::RowSetOps;
|
||||
|
||||
let scope: Option<roaring::RoaringBitmap> =
|
||||
scope.map(|ids| ids.iter().map(|id| *id as u32).collect());
|
||||
let segments = metadata_index::segments(dataset, &indexed.column).await?;
|
||||
let query = LabelListQuery::HasAnyLabel(
|
||||
indexed
|
||||
.values
|
||||
.iter()
|
||||
.map(|v| ScalarValue::Utf8(Some(v.clone())))
|
||||
.collect(),
|
||||
);
|
||||
let mut searches: Vec<futures::future::BoxFuture<'_, Result<u64>>> = Vec::new();
|
||||
for segment in segments.iter().rev() {
|
||||
let fragments = match &scope {
|
||||
Some(scope) => &segment.fragments & scope,
|
||||
None => segment.fragments.clone(),
|
||||
};
|
||||
if fragments.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let uuid = segment.uuid;
|
||||
let query = &query;
|
||||
searches.push(Box::pin(async move {
|
||||
let index = dataset
|
||||
.open_scalar_index(&indexed.column, &uuid, &NoOpMetricsCollector)
|
||||
.await?;
|
||||
let mut found = match index.search(query, &NoOpMetricsCollector).await? {
|
||||
SearchResult::Exact(rows) | SearchResult::AtMost(rows) => rows.true_rows(),
|
||||
SearchResult::AtLeast(_) => anyhow::bail!("a LABEL_LIST index answered AtLeast"),
|
||||
};
|
||||
found.retain_fragments(fragments.iter());
|
||||
Ok(found.len().unwrap_or(u64::MAX))
|
||||
}));
|
||||
}
|
||||
let mut counts = futures::stream::iter(searches).buffered(PARALLEL_SEGMENTS);
|
||||
let mut total = 0u64;
|
||||
while let Some(count) = counts.try_next().await? {
|
||||
total = total.saturating_add(count);
|
||||
if total > limit {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let unindexed: Vec<lance_table::format::Fragment> = dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.filter(|f| !segments.iter().any(|s| s.fragments.contains(f.id as u32)))
|
||||
.filter(|f| scope.as_ref().is_none_or(|s| s.contains(f.id as u32)))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !unindexed.is_empty() {
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.filter_expr(indexed.expr.clone())
|
||||
.with_fragments(unindexed);
|
||||
total = total.saturating_add(scanner.count_rows().await?);
|
||||
}
|
||||
Ok((total <= limit).then_some(total))
|
||||
}
|
||||
|
||||
/// How a filter compares a key.
|
||||
pub mod operand {
|
||||
pub const STRING: u8 = 1;
|
||||
@@ -450,7 +546,7 @@ mod tests {
|
||||
let indexed = crate::filter::apply(&mut scanner, filter, &ready_columns(dataset)).unwrap();
|
||||
(
|
||||
scanner.get_expr_filter().unwrap().unwrap().to_string(),
|
||||
indexed.map(|e| e.to_string()),
|
||||
indexed.first().map(|i| i.expr.to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -596,4 +692,48 @@ mod tests {
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/// Counting walks the segments, keeps to the scope and stops once over
|
||||
/// the limit.
|
||||
#[tokio::test]
|
||||
async fn counts_follow_segments_and_scope() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut dataset = dataset(&dir, 1000).await;
|
||||
dataset.drop_index("_mk_0_idx").await.unwrap();
|
||||
// Ten fragments of 100 rows: four segments of up to 300 rows.
|
||||
let ids: Vec<u64> = dataset.fragments().iter().map(|f| f.id).collect();
|
||||
metadata_index::index_fragments(&mut dataset, "_mk_0", 300, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let segments = metadata_index::segments(&dataset, "_mk_0").await.unwrap();
|
||||
assert_eq!(segments.len(), 4);
|
||||
let ready = ready_columns(&dataset);
|
||||
let indexed =
|
||||
crate::filter::apply(&mut dataset.scan(), &eq("author", "\"rare\""), &ready).unwrap();
|
||||
let rare = &indexed[0];
|
||||
// "rare" in rows i % 10 == 0, "rare2" never matches it.
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, None, 1_000).await.unwrap(),
|
||||
Some(100)
|
||||
);
|
||||
assert_eq!(count_indexed(&dataset, rare, None, 99).await.unwrap(), None);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&ids[..2]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(20)
|
||||
);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&ids[9..]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(10)
|
||||
);
|
||||
assert_eq!(
|
||||
count_indexed(&dataset, rare, Some(&[]), 1_000)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(0)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -139,24 +139,22 @@ pub(crate) async fn few_indexed_matches(
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(indexed) = crate::filter::apply(&mut dataset.scan(), filter, &ready)? else {
|
||||
let indexed = crate::filter::apply(&mut dataset.scan(), filter, &ready)?;
|
||||
if indexed.is_empty() {
|
||||
return Ok(false);
|
||||
};
|
||||
let mut counter = dataset.scan();
|
||||
counter.filter_expr(indexed);
|
||||
if let Some(ids) = scope(dataset, table, query).await? {
|
||||
counter.with_fragments(
|
||||
dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.filter(|f| ids.contains(&f.id))
|
||||
.cloned()
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
let matches = counter.count_rows().await?;
|
||||
tracing::debug!(matches, limit, "indexed metadata filter");
|
||||
Ok(matches <= limit as u64)
|
||||
let scope = scope(dataset, table, query).await?;
|
||||
// An AND matches at most what its most selective indexed conjunct does.
|
||||
for conjunct in &indexed {
|
||||
let count =
|
||||
crate::indexed_filter::count_indexed(dataset, conjunct, scope.as_deref(), limit as u64)
|
||||
.await?;
|
||||
tracing::debug!(column = %conjunct.column, ?count, limit, "indexed metadata filter");
|
||||
if count.is_some() {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fetch rows by key, projected to `columns` (all of them when None).
|
||||
|
||||
@@ -16,6 +16,7 @@ const BUCKET: Duration = Duration::from_secs(300);
|
||||
pub const WINDOW: Duration = Duration::from_secs(BUCKET.as_secs() * BUCKETS as u64);
|
||||
pub const MAX_INDEXES: usize = 512;
|
||||
pub const MAX_KEYS_PER_INDEX: usize = 32;
|
||||
const MAX_RETIRED_TABLES: usize = 4;
|
||||
|
||||
#[derive(Default, Clone, Copy)]
|
||||
struct Bucket {
|
||||
@@ -34,12 +35,36 @@ struct KeyStats {
|
||||
|
||||
#[derive(Default)]
|
||||
struct IndexStats {
|
||||
/// The table the stats were counted on: an index dropped and created
|
||||
/// again under its name is another table.
|
||||
table: String,
|
||||
/// Tables the name held before, whose late queries are not counted.
|
||||
retired: Vec<String>,
|
||||
keys: HashMap<String, KeyStats>,
|
||||
/// The last bucket a key was left uncounted in.
|
||||
truncated: Option<u64>,
|
||||
last: u64,
|
||||
}
|
||||
|
||||
impl IndexStats {
|
||||
/// Starts counting on `table`, retiring the table counted on before.
|
||||
fn replace(&mut self, table: &str) {
|
||||
let mut retired = std::mem::take(&mut self.retired);
|
||||
if !self.table.is_empty() {
|
||||
retired.push(std::mem::take(&mut self.table));
|
||||
if retired.len() > MAX_RETIRED_TABLES {
|
||||
retired.remove(0);
|
||||
}
|
||||
}
|
||||
*self = IndexStats {
|
||||
table: table.to_string(),
|
||||
retired,
|
||||
last: self.last,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether bucket `at` is still in the window of bucket `now`.
|
||||
fn live(at: u64, now: u64) -> bool {
|
||||
at + BUCKETS as u64 > now
|
||||
@@ -68,12 +93,37 @@ impl FilterStats {
|
||||
(now.duration_since(self.start).as_secs() / BUCKET.as_secs()) + 1
|
||||
}
|
||||
|
||||
/// One query of `index` that took `elapsed` and filtered on `keys`.
|
||||
pub fn record(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration) {
|
||||
self.record_at(index, keys, elapsed, Instant::now());
|
||||
/// Drops the stats of `index`, which now holds `table` ("" once
|
||||
/// dropped): late queries of the table it held before are not counted.
|
||||
pub fn forget(&self, index: &str, table: &str) {
|
||||
let mut indexes = self.indexes.lock().unwrap();
|
||||
if let Some(stats) = indexes.get_mut(index)
|
||||
&& (table.is_empty() || stats.table != table)
|
||||
{
|
||||
stats.replace(table);
|
||||
}
|
||||
}
|
||||
|
||||
/// One query of `index`, whose table is `table`, that took `elapsed` and
|
||||
/// filtered on `keys`. Stats counted on another table under the name are
|
||||
/// dropped; a query of a table the name no longer holds is not counted.
|
||||
pub fn record(&self, index: &str, table: &str, keys: &BTreeMap<String, u8>, elapsed: Duration) {
|
||||
self.record_table_at(index, table, keys, elapsed, Instant::now());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_at(&self, index: &str, keys: &BTreeMap<String, u8>, elapsed: Duration, now: Instant) {
|
||||
self.record_table_at(index, "", keys, elapsed, now);
|
||||
}
|
||||
|
||||
fn record_table_at(
|
||||
&self,
|
||||
index: &str,
|
||||
table: &str,
|
||||
keys: &BTreeMap<String, u8>,
|
||||
elapsed: Duration,
|
||||
now: Instant,
|
||||
) {
|
||||
if keys.is_empty() {
|
||||
return;
|
||||
}
|
||||
@@ -90,6 +140,16 @@ impl FilterStats {
|
||||
indexes.remove(&oldest);
|
||||
}
|
||||
let stats = indexes.entry(index.to_string()).or_default();
|
||||
if !table.is_empty() && stats.table != table {
|
||||
if stats.retired.iter().any(|t| t == table) {
|
||||
return;
|
||||
}
|
||||
if stats.table.is_empty() {
|
||||
stats.table = table.to_string();
|
||||
} else {
|
||||
stats.replace(table);
|
||||
}
|
||||
}
|
||||
stats.last = stats.last.max(at);
|
||||
for (key, kinds) in keys {
|
||||
if !stats.keys.contains_key(key) && stats.keys.len() >= MAX_KEYS_PER_INDEX {
|
||||
@@ -287,4 +347,46 @@ mod tests {
|
||||
);
|
||||
assert_eq!(stats.indexes.lock().unwrap()["b/i"].last, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_follow_the_table_behind_an_index_name() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
let author = keys(&[("author", operand::STRING)]);
|
||||
let count = |stats: &FilterStats| {
|
||||
stats
|
||||
.snapshot_at("b/i", t0)
|
||||
.keys
|
||||
.first()
|
||||
.map_or(0, |k| k.queries)
|
||||
};
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 2);
|
||||
// The index was created again: its first query starts over.
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
// A late query of the old table is not counted on the new one.
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
stats.forget("b/i", "");
|
||||
assert_eq!(count(&stats), 0);
|
||||
// A first query records its table: a later table starts over.
|
||||
stats.record_table_at("b/i", "t3", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t4", &author, Duration::ZERO, t0);
|
||||
assert_eq!(count(&stats), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_late_query_after_a_drop_is_not_counted_on_the_new_table() {
|
||||
let stats = FilterStats::new();
|
||||
let t0 = stats.start;
|
||||
let author = keys(&[("author", operand::STRING)]);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
stats.forget("b/i", "");
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t2", &author, Duration::ZERO, t0);
|
||||
stats.record_table_at("b/i", "t1", &author, Duration::ZERO, t0);
|
||||
assert_eq!(stats.snapshot_at("b/i", t0).keys[0].queries, 2);
|
||||
}
|
||||
}
|
||||
@@ -389,12 +389,27 @@ impl Fixture {
|
||||
|
||||
fn assert_same(got: &[(String, f32)], want: &[(String, f32)], what: &str) {
|
||||
assert_eq!(got.len(), want.len(), "{what}: {got:?} vs {want:?}");
|
||||
// Ties may come back in either order: the same keys, and the same
|
||||
// distance at each rank.
|
||||
fn keys(hits: &[(String, f32)]) -> Vec<&str> {
|
||||
let mut keys: Vec<&str> = hits.iter().map(|(k, _)| k.as_str()).collect();
|
||||
keys.sort();
|
||||
keys
|
||||
}
|
||||
assert_eq!(keys(got), keys(want), "{what}: {got:?} vs {want:?}");
|
||||
for (g, w) in got.iter().zip(want) {
|
||||
let same = g.0 == w.0 || (g.1 - w.1).abs() < 1e-4;
|
||||
assert!(
|
||||
same && ((g.1 - w.1).abs() < 1e-4 || (g.1.is_infinite() && w.1.is_infinite())),
|
||||
"{what}: {got:?} vs {want:?}"
|
||||
);
|
||||
let same = (g.1 - w.1).abs() < 1e-4 || (g.1.is_infinite() && w.1.is_infinite());
|
||||
assert!(same, "{what}: {got:?} vs {want:?}");
|
||||
if g.0 != w.0 {
|
||||
let tie = want
|
||||
.iter()
|
||||
.any(|(k, d)| k == &g.0 && (d - w.1).abs() < 1e-4);
|
||||
assert!(
|
||||
tie,
|
||||
"{what}: {} at {} is no tie: {got:?} vs {want:?}",
|
||||
g.0, g.1
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -564,6 +579,29 @@ async fn a_write_that_raced_the_column_add_fills_the_column() {
|
||||
assert_eq!(fixture.flat(latest, filter, 400, &v).await.len(), 4);
|
||||
}
|
||||
|
||||
/// A write names no version: one at a version from before a key's column
|
||||
/// add would commit rows without the column.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_write_at_an_older_version_is_refused() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..10).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let before = fixture.dataset(0).await.version().version;
|
||||
fixture.index_author().await;
|
||||
let mut table = fixture.table.clone();
|
||||
table.version = before;
|
||||
let refused = fixture
|
||||
.server
|
||||
.put_vectors(Request::new(pb::PutVectorsRequest {
|
||||
table: Some(table),
|
||||
vectors: (10..20).map(|i| row(i, author(i))).collect(),
|
||||
}))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(refused.code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
|
||||
/// Two writes updating the same keys raced the column add, and the first
|
||||
/// one's fill commits last: each write's rows get the values of its own
|
||||
/// metadata, not the ones derived before it or from the other write.
|
||||
@@ -625,3 +663,155 @@ async fn a_write_that_raced_the_column_add_derives_its_updated_rows() {
|
||||
got.sort_by_key(|v| v.key[1..].parse::<usize>().unwrap());
|
||||
assert_eq!(got, rows("bob"));
|
||||
}
|
||||
|
||||
/// Operators ask through the engine; the table records the request and,
|
||||
/// once the worker has done its part, the key's state, coverage and size.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn keys_are_requested_and_listed_through_the_engine() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..600).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let update = |key: &str, remove: bool| {
|
||||
fixture
|
||||
.server
|
||||
.update_metadata_key(Request::new(pb::UpdateMetadataKeyRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
key: key.to_string(),
|
||||
remove,
|
||||
}))
|
||||
};
|
||||
let listed = update("author", false).await.unwrap().into_inner();
|
||||
assert_eq!(listed.keys.len(), 1);
|
||||
assert_eq!(listed.keys[0].state, "requested");
|
||||
assert_eq!(listed.keys[0].rows, 600);
|
||||
// A retry of a committed request answers as the first one did.
|
||||
let again = update("author", false).await.unwrap().into_inner();
|
||||
assert_eq!(again.keys.len(), 1);
|
||||
assert_eq!(again.keys[0].state, "requested");
|
||||
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
let key = metadata_index::add_column(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::finish_after(&mut dataset, &key, 1_000_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
let listed = fixture
|
||||
.server
|
||||
.metadata_keys(Request::new(pb::MetadataKeysRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
assert_eq!((listed.rows, listed.vector_index.as_str()), (600, ""));
|
||||
let author = &listed.keys[0];
|
||||
assert_eq!(
|
||||
(
|
||||
author.key.as_str(),
|
||||
author.state.as_str(),
|
||||
author.column.as_str()
|
||||
),
|
||||
("author", "ready", "_mk_0")
|
||||
);
|
||||
assert_eq!(
|
||||
(author.indexed_rows, author.rows, author.segments),
|
||||
(600, 600, 1)
|
||||
);
|
||||
assert!(author.index_bytes > 0);
|
||||
|
||||
// Off at once: the next query reads the JSON again.
|
||||
let listed = update("author", true).await.unwrap().into_inner();
|
||||
assert_eq!(listed.keys[0].state, "off");
|
||||
let dataset = fixture.dataset(0).await;
|
||||
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
|
||||
// Removing a key that is not indexed needs nothing.
|
||||
let nobody = update("nobody", true).await.unwrap().into_inner();
|
||||
assert_eq!(nobody.keys.len(), 1);
|
||||
assert_eq!(
|
||||
update("author", false).await.unwrap_err().code(),
|
||||
tonic::Code::FailedPrecondition,
|
||||
"being removed"
|
||||
);
|
||||
}
|
||||
|
||||
/// DeleteIndex purges the table's directory and an index created again
|
||||
/// under the same name reuses it: the same running engine serves nothing of
|
||||
/// the old table, neither rows, nor its indexed key, nor its filter stats.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_dropped_and_recreated_index_serves_nothing_old() {
|
||||
let fixture = Fixture::new().await;
|
||||
fixture
|
||||
.put((0..600).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let mut dataset = fixture.dataset(0).await;
|
||||
weed_lance_worker::vector_index::build(&mut dataset, "vector", "vector_idx", 4, false)
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.index_author().await;
|
||||
let bob = FILTERS[1].1;
|
||||
let old = fixture
|
||||
.query(&fixture.server, 0, bob, 50, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(old.len(), 6);
|
||||
let server = &fixture.server;
|
||||
let stats = || {
|
||||
server.filter_stats(Request::new(pb::FilterStatsRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
};
|
||||
assert_eq!(stats().await.unwrap().into_inner().keys[0].queries, 1);
|
||||
|
||||
// What the catalog's DeleteTable does, then CreateIndex again.
|
||||
let (store, path) = lance::io::ObjectStore::from_uri(&fixture.table.location)
|
||||
.await
|
||||
.unwrap();
|
||||
store.remove_dir_all(path).await.unwrap();
|
||||
fixture
|
||||
.server
|
||||
.create_index(Request::new(pb::CreateIndexRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
dimension: DIM as u32,
|
||||
distance_metric: pb::DistanceMetric::L2 as i32,
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = fixture
|
||||
.server
|
||||
.metadata_keys(Request::new(pb::MetadataKeysRequest {
|
||||
table: Some(fixture.table.clone()),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
assert!(keys.keys.is_empty());
|
||||
assert!(
|
||||
stats().await.unwrap().into_inner().keys.is_empty(),
|
||||
"the old table's filter stats"
|
||||
);
|
||||
|
||||
fixture
|
||||
.put((1000..1100).map(|i| row(i, author(i))).collect())
|
||||
.await;
|
||||
let new = |hits: &[(String, f32)]| {
|
||||
hits.iter().all(|(k, _)| {
|
||||
k[1..]
|
||||
.parse::<usize>()
|
||||
.is_ok_and(|i| (1000..1100).contains(&i))
|
||||
})
|
||||
};
|
||||
let unfiltered = fixture
|
||||
.query(&fixture.server, 0, "", 200, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(unfiltered.len(), 100);
|
||||
assert!(new(&unfiltered), "{unfiltered:?}");
|
||||
let bob_now = fixture
|
||||
.query(&fixture.server, 0, bob, 50, &vector_of(10))
|
||||
.await;
|
||||
assert_eq!(bob_now.len(), 1);
|
||||
assert!(new(&bob_now), "{bob_now:?}");
|
||||
let author = stats().await.unwrap().into_inner().keys;
|
||||
assert_eq!((author.len(), author[0].queries), (1, 1));
|
||||
}
|
||||
@@ -78,6 +78,24 @@ impl VectorService for ObservedService {
|
||||
}
|
||||
result
|
||||
}
|
||||
async fn metadata_keys(
|
||||
&self,
|
||||
request: Request<pb::MetadataKeysRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let mut in_flight = self.metrics.start("MetadataKeys");
|
||||
let result = self.inner.metadata_keys(request).await;
|
||||
in_flight.complete(result.as_ref().err().map(|e| e.code()));
|
||||
result
|
||||
}
|
||||
async fn update_metadata_key(
|
||||
&self,
|
||||
request: Request<pb::UpdateMetadataKeyRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let mut in_flight = self.metrics.start("UpdateMetadataKey");
|
||||
let result = self.inner.update_metadata_key(request).await;
|
||||
in_flight.complete(result.as_ref().err().map(|e| e.code()));
|
||||
result
|
||||
}
|
||||
async fn filter_stats(
|
||||
&self,
|
||||
request: Request<pb::FilterStatsRequest>,
|
||||
|
||||
@@ -328,6 +328,50 @@ pub(crate) async fn heal_columns(
|
||||
bail!("writes kept replacing the rows while their indexed key columns were derived")
|
||||
}
|
||||
|
||||
/// The indexed metadata keys `dataset` records, requested ones first.
|
||||
async fn metadata_keys(dataset: &Dataset) -> Result<pb::MetadataKeysResponse> {
|
||||
let rows: u64 = dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.map(|f| f.num_rows().unwrap_or(0) as u64)
|
||||
.sum();
|
||||
let mut keys: Vec<pb::MetadataKey> = metadata_index::requests(dataset)
|
||||
.into_iter()
|
||||
.map(|key| pb::MetadataKey {
|
||||
key,
|
||||
state: "requested".to_string(),
|
||||
rows,
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
for key in metadata_index::keys_of(dataset) {
|
||||
let segments = metadata_index::segments(dataset, &key.column).await?;
|
||||
keys.push(pb::MetadataKey {
|
||||
state: key.state.map_or("unknown", |s| s.as_str()).to_string(),
|
||||
indexed_rows: segments.iter().map(|s| s.rows).sum(),
|
||||
index_bytes: segments.iter().map(|s| s.bytes).sum(),
|
||||
segments: segments.len() as u32,
|
||||
rows,
|
||||
key: key.key,
|
||||
column: key.column,
|
||||
});
|
||||
}
|
||||
let vector_index = match index::index_names(dataset, VECTOR_COLUMN).await?.first() {
|
||||
Some(name) => {
|
||||
weed_lance_worker::vector_index::index_settings(dataset, name)
|
||||
.await?
|
||||
.index_type
|
||||
}
|
||||
None => String::new(),
|
||||
};
|
||||
Ok(pb::MetadataKeysResponse {
|
||||
keys,
|
||||
version: dataset.version().version,
|
||||
rows,
|
||||
vector_index,
|
||||
})
|
||||
}
|
||||
|
||||
fn struct_to_json(s: &Struct) -> serde_json::Value {
|
||||
serde_json::Value::Object(
|
||||
s.fields
|
||||
@@ -885,7 +929,11 @@ impl VectorService for VectorServer {
|
||||
)
|
||||
.await;
|
||||
let dataset = match dataset {
|
||||
Ok(dataset) => dataset,
|
||||
Ok(dataset) => {
|
||||
self.filter_stats
|
||||
.forget(&table.id.join("/"), table_id(&dataset));
|
||||
dataset
|
||||
}
|
||||
Err(error) => match self.open(&table).await {
|
||||
Ok(existing)
|
||||
if matches_index_schema(
|
||||
@@ -936,6 +984,7 @@ impl VectorService for VectorServer {
|
||||
store.remove_dir_all(path).await.map_err(internal)?;
|
||||
}
|
||||
}
|
||||
self.filter_stats.forget(&table.id.join("/"), "");
|
||||
Ok(Response::new(pb::DropIndexResponse {}))
|
||||
}
|
||||
|
||||
@@ -947,13 +996,37 @@ impl VectorService for VectorServer {
|
||||
let table = req
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
let dataset = self.open(&table).await?;
|
||||
if req.vectors.is_empty() {
|
||||
return Ok(Response::new(pb::PutVectorsResponse {
|
||||
version: dataset.version().version,
|
||||
}));
|
||||
if table.version != 0 {
|
||||
return Err(Status::invalid_argument("a write takes no table version"));
|
||||
}
|
||||
let dataset = write_vectors(dataset, &req.vectors).await?;
|
||||
// A write commits within its deadline of opening the table or not
|
||||
// at all: a key's column add waits longer than that before the key
|
||||
// is ready, so a write that opened before the add has committed,
|
||||
// and been derived again, before any query uses the key.
|
||||
let merged = tokio::time::timeout(metadata_index::WRITE_DEADLINE, async {
|
||||
let dataset = self.open(&table).await?;
|
||||
if req.vectors.is_empty() {
|
||||
return Ok::<_, Status>((dataset, None));
|
||||
}
|
||||
let written = Arc::new(Into::<arrow_schema::Schema>::into(dataset.schema()));
|
||||
Ok((merge_vectors(dataset, &req.vectors).await?, Some(written)))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
Status::deadline_exceeded(format!(
|
||||
"PutVectors took longer than {:?} to commit",
|
||||
metadata_index::WRITE_DEADLINE
|
||||
))
|
||||
})??;
|
||||
let dataset = match merged {
|
||||
(dataset, Some(written)) => {
|
||||
let keys: Vec<String> = req.vectors.iter().map(|v| v.key.clone()).collect();
|
||||
heal_columns(dataset, &written, &keys)
|
||||
.await
|
||||
.map_err(internal)?
|
||||
}
|
||||
(dataset, None) => dataset,
|
||||
};
|
||||
Ok(Response::new(pb::PutVectorsResponse {
|
||||
version: dataset.version().version,
|
||||
}))
|
||||
@@ -1186,11 +1259,47 @@ impl VectorService for VectorServer {
|
||||
data: data_map.remove(&hit.key).unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
let index = table.id.join("/");
|
||||
self.filter_stats
|
||||
.record(&table.id.join("/"), &filter_keys, started.elapsed());
|
||||
.record(&index, table_id(&dataset), &filter_keys, started.elapsed());
|
||||
Ok(Response::new(pb::QueryVectorsResponse { vectors }))
|
||||
}
|
||||
|
||||
async fn metadata_keys(
|
||||
&self,
|
||||
request: Request<pb::MetadataKeysRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let table = request
|
||||
.into_inner()
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
let dataset = self.open_to_read(&table).await?;
|
||||
let _lease = self.leases.try_pin(&dataset).await.map_err(internal)?;
|
||||
Ok(Response::new(
|
||||
metadata_keys(&dataset).await.map_err(internal)?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_metadata_key(
|
||||
&self,
|
||||
request: Request<pb::UpdateMetadataKeyRequest>,
|
||||
) -> Result<Response<pb::MetadataKeysResponse>, Status> {
|
||||
let req = request.into_inner();
|
||||
let table = req
|
||||
.table
|
||||
.ok_or_else(|| Status::invalid_argument("missing table"))?;
|
||||
let mut dataset = self.open(&table).await?;
|
||||
let updated = if req.remove {
|
||||
metadata_index::request_removal(&mut dataset, &req.key).await
|
||||
} else {
|
||||
metadata_index::request(&mut dataset, &req.key).await
|
||||
};
|
||||
updated.map_err(|e| Status::failed_precondition(format!("{e:#}")))?;
|
||||
Ok(Response::new(
|
||||
metadata_keys(&dataset).await.map_err(internal)?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn filter_stats(
|
||||
&self,
|
||||
request: Request<pb::FilterStatsRequest>,
|
||||
|
||||
Generated
+2
@@ -7157,6 +7157,7 @@ dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -7205,6 +7206,7 @@ dependencies = [
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"x509-parser",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
@@ -38,11 +38,13 @@ lance-datafusion = "10"
|
||||
arrow-array = "58"
|
||||
arrow-schema = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-select = "58"
|
||||
# Lance's stored form of the metadata JSON, read when deriving the
|
||||
# indexed-key columns.
|
||||
jsonb = { version = "0.5", default-features = false }
|
||||
chrono = "0.4"
|
||||
futures = "0.3"
|
||||
uuid = "1"
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
@@ -95,6 +97,7 @@ tonic.workspace = true
|
||||
lance-arrow = "10"
|
||||
arrow-schema = "58"
|
||||
arrow-cast = "58"
|
||||
arrow-select = "58"
|
||||
lance-linalg = "10"
|
||||
tempfile = "3"
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ use lance::io::{
|
||||
use lance::session::Session;
|
||||
|
||||
use crate::catalog::{NamespaceClient, TableDescription};
|
||||
use crate::direct_read::DirectReads;
|
||||
use crate::direct_read::{DirectReads, Wrappers};
|
||||
use crate::drop_guard::DropGuard;
|
||||
|
||||
/// A table the worker is about to work on.
|
||||
pub struct OpenTable {
|
||||
@@ -75,7 +76,7 @@ pub fn unsigned_options() -> HashMap<String, String> {
|
||||
pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) -> Result<OpenTable> {
|
||||
let description = client.describe_table(id).await?;
|
||||
let wrapper = storage.wrapper(&description.location).await?;
|
||||
let dataset = open_at(&description, wrapper, None).await?;
|
||||
let dataset = open_guarded(&description, wrapper).await?;
|
||||
Ok(OpenTable {
|
||||
id: id.to_vec(),
|
||||
location: description.location,
|
||||
@@ -83,6 +84,21 @@ pub async fn open(client: &NamespaceClient, id: &[String], storage: &Storage) ->
|
||||
})
|
||||
}
|
||||
|
||||
/// [`open_at`] for a job that writes the table: its writes fail once the
|
||||
/// table is dropped under it (see [`DropGuard`]).
|
||||
pub async fn open_guarded(
|
||||
description: &TableDescription,
|
||||
wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||
) -> Result<Dataset> {
|
||||
let guard = DropGuard::new();
|
||||
let mut wrappers: Vec<Arc<dyn WrappingObjectStore>> = wrapper.into_iter().collect();
|
||||
wrappers.push(guard.clone());
|
||||
let dataset = open_at(description, Some(Arc::new(Wrappers(wrappers))), None).await?;
|
||||
let opened = dataset.manifest_location();
|
||||
guard.arm(opened.path.clone(), opened.e_tag.clone());
|
||||
Ok(dataset)
|
||||
}
|
||||
|
||||
/// Open a table the caller has already resolved, with the store wrapped, in
|
||||
/// `session` when given, otherwise in a session of its own.
|
||||
pub async fn open_at(
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
//! Keeps a job from writing into a table dropped under it.
|
||||
//!
|
||||
//! DeleteIndex purges a table's directory, and an index created again under
|
||||
//! the same name reuses it. A job that opened the table before the purge
|
||||
//! would otherwise go on writing there: its data files would land in the
|
||||
//! new table's directory, and its commit would put a manifest of the old
|
||||
//! table there, so the new table could not be created, or would read it.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::BoxStream;
|
||||
use lance::io::WrappingObjectStore;
|
||||
use object_store::path::Path;
|
||||
use object_store::{
|
||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
|
||||
UploadPart,
|
||||
};
|
||||
|
||||
/// Fails every write once the manifest it watches is gone or is another
|
||||
/// object (a table created again reuses version paths): the one of the
|
||||
/// version the job opened, then the newest the job committed. Armed after the
|
||||
/// open; until then it lets everything through. When the job itself deletes
|
||||
/// the watched manifest, removing old versions as a purge does, it watches
|
||||
/// the newest manifest left instead.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DropGuard {
|
||||
watched: Arc<Mutex<Watched>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct Watched {
|
||||
manifest: Option<Path>,
|
||||
/// The manifest's e-tag when known, telling it from a replacement.
|
||||
e_tag: Option<String>,
|
||||
/// The job deleted the watched manifest itself.
|
||||
deleted: bool,
|
||||
}
|
||||
|
||||
impl DropGuard {
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self::default())
|
||||
}
|
||||
|
||||
/// Watch `manifest`, the opened version's, with its e-tag when known.
|
||||
pub fn arm(&self, manifest: Path, e_tag: Option<String>) {
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
watched.manifest = Some(manifest);
|
||||
watched.e_tag = e_tag;
|
||||
}
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for DropGuard {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(Guarded {
|
||||
inner: original,
|
||||
watched: self.watched.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Guarded {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
watched: Arc<Mutex<Watched>>,
|
||||
}
|
||||
|
||||
/// What a check found: the table is there, or it was dropped, or created
|
||||
/// again, with the error to fail the write with.
|
||||
enum Found {
|
||||
Table,
|
||||
Dropped(object_store::Error),
|
||||
Replaced(object_store::Error),
|
||||
}
|
||||
|
||||
impl Guarded {
|
||||
/// Whether the watched manifest is still the table's. An error is a
|
||||
/// failed probe, which says nothing about the table.
|
||||
async fn find(&self, location: &Path) -> Result<Found> {
|
||||
let watched = self.watched.lock().unwrap().clone();
|
||||
let Some(mut manifest) = watched.manifest else {
|
||||
return Ok(Found::Table);
|
||||
};
|
||||
let mut e_tag = watched.e_tag;
|
||||
if watched.deleted {
|
||||
if let Some((newest, tag)) = self.newest_manifest(&manifest).await? {
|
||||
manifest = newest;
|
||||
e_tag = tag;
|
||||
}
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
watched.manifest = Some(manifest.clone());
|
||||
watched.e_tag = e_tag.clone();
|
||||
watched.deleted = false;
|
||||
}
|
||||
let error = |why: &str| {
|
||||
object_store::Error::Generic {
|
||||
store: "DropGuard",
|
||||
source: format!(
|
||||
"the table was dropped while this job wrote it: {manifest} {why}, so {location} is not written"
|
||||
)
|
||||
.into(),
|
||||
}
|
||||
};
|
||||
match self.inner.head(&manifest).await {
|
||||
Ok(meta) => match (&e_tag, &meta.e_tag) {
|
||||
(Some(want), Some(have)) if want != have => {
|
||||
Ok(Found::Replaced(error("is another table's")))
|
||||
}
|
||||
_ => Ok(Found::Table),
|
||||
},
|
||||
// Gone with manifests beside it: a table created again, at an
|
||||
// earlier version than the one watched.
|
||||
Err(object_store::Error::NotFound { .. }) => {
|
||||
match self.newest_manifest(&manifest).await? {
|
||||
Some(_) => Ok(Found::Replaced(error("is gone and another table is there"))),
|
||||
None => Ok(Found::Dropped(error("is gone"))),
|
||||
}
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn check(&self, location: &Path) -> Result<()> {
|
||||
match self.find(location).await? {
|
||||
Found::Table => Ok(()),
|
||||
Found::Dropped(error) | Found::Replaced(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_manifest(location: &Path) -> bool {
|
||||
location.extension() == Some("manifest")
|
||||
}
|
||||
|
||||
impl Guarded {
|
||||
/// The most recently written manifest beside `manifest`, with its e-tag.
|
||||
async fn newest_manifest(&self, manifest: &Path) -> Result<Option<(Path, Option<String>)>> {
|
||||
let dir: Path = manifest
|
||||
.parts()
|
||||
.take(manifest.parts().count() - 1)
|
||||
.collect();
|
||||
let listed = self.inner.list_with_delimiter(Some(&dir)).await?;
|
||||
Ok(listed
|
||||
.objects
|
||||
.into_iter()
|
||||
.filter(|o| is_manifest(&o.location))
|
||||
.max_by_key(|o| o.last_modified)
|
||||
.map(|o| (o.location, o.e_tag)))
|
||||
}
|
||||
|
||||
fn committed(&self, location: &Path, e_tag: Option<String>) {
|
||||
if is_manifest(location) {
|
||||
let mut watched = self.watched.lock().unwrap();
|
||||
if watched.manifest.is_some() {
|
||||
watched.manifest = Some(location.clone());
|
||||
watched.e_tag = e_tag;
|
||||
watched.deleted = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A multipart upload that checks the table again before it completes.
|
||||
#[derive(Debug)]
|
||||
struct GuardedUpload {
|
||||
inner: Box<dyn MultipartUpload>,
|
||||
guard: Guarded,
|
||||
location: Path,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MultipartUpload for GuardedUpload {
|
||||
fn put_part(&mut self, data: PutPayload) -> UploadPart {
|
||||
self.inner.put_part(data)
|
||||
}
|
||||
|
||||
async fn complete(&mut self) -> Result<PutResult> {
|
||||
// A failed probe aborts too: the parts would outlive the upload.
|
||||
let error = match self.guard.find(&self.location).await {
|
||||
Ok(Found::Table) => return self.inner.complete().await,
|
||||
Ok(Found::Dropped(error) | Found::Replaced(error)) | Err(error) => error,
|
||||
};
|
||||
let _ = self.inner.abort().await;
|
||||
Err(error)
|
||||
}
|
||||
|
||||
async fn abort(&mut self) -> Result<()> {
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Guarded {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "DropGuard({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for Guarded {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
self.check(location).await?;
|
||||
let put = self.inner.put_opts(location, payload, opts).await?;
|
||||
// A drop can land between the check and the write. A manifest is
|
||||
// what makes the table readable again, so look once more and take
|
||||
// it back if the table is gone; a failed probe leaves it. A table
|
||||
// created again since may have written its own manifest there:
|
||||
// only a manifest known to be this write's is taken back.
|
||||
if is_manifest(location) {
|
||||
let (error, replaced) = match self.find(location).await {
|
||||
Ok(Found::Dropped(error)) => (error, false),
|
||||
Ok(Found::Replaced(error)) => (error, true),
|
||||
_ => {
|
||||
self.committed(location, put.e_tag.clone());
|
||||
return Ok(put);
|
||||
}
|
||||
};
|
||||
let ours = match (&put.e_tag, self.inner.head(location).await) {
|
||||
(Some(wrote), Ok(meta)) => meta.e_tag.as_ref() == Some(wrote),
|
||||
(_, Err(object_store::Error::NotFound { .. })) => false,
|
||||
_ => !replaced,
|
||||
};
|
||||
if ours {
|
||||
let _ = self.inner.delete(location).await;
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
self.committed(location, put.e_tag.clone());
|
||||
Ok(put)
|
||||
}
|
||||
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
self.check(location).await?;
|
||||
let inner = self.inner.put_multipart_opts(location, opts).await?;
|
||||
Ok(Box::new(GuardedUpload {
|
||||
inner,
|
||||
guard: self.clone(),
|
||||
location: location.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
// A cleanup picks its paths before it deletes them; a table created
|
||||
// again meanwhile reuses those paths, so look once before deleting.
|
||||
// In a dropped table there is nothing left to spare, and the job's
|
||||
// own failed commit is cleaned up.
|
||||
let watched = self.watched.clone();
|
||||
let locations = locations.inspect(move |location| {
|
||||
if let Ok(location) = location {
|
||||
let mut watched = watched.lock().unwrap();
|
||||
if watched.manifest.as_ref() == Some(location) {
|
||||
watched.deleted = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
let guard = self.clone();
|
||||
let pending = Arc::new(Mutex::new(Some(locations.boxed())));
|
||||
let checked =
|
||||
futures::stream::once(async move { guard.find(&Path::from("cleanup")).await })
|
||||
.flat_map(move |found| {
|
||||
let locations = pending.lock().unwrap().take();
|
||||
match (found, locations) {
|
||||
(Ok(Found::Table | Found::Dropped(_)), Some(locations)) => locations,
|
||||
(Ok(Found::Replaced(error)), _) | (Err(error), _) => {
|
||||
futures::stream::once(async move { Err(error) }).boxed()
|
||||
}
|
||||
(Ok(_), None) => futures::stream::empty().boxed(),
|
||||
}
|
||||
})
|
||||
.boxed();
|
||||
self.inner.delete_stream(checked)
|
||||
}
|
||||
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.check(to).await?;
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.check(to).await?;
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,701 @@
|
||||
//! A table dropped while a worker job runs on it: the job must fail without
|
||||
//! writing into the dropped path, and a table created again there starts
|
||||
//! empty.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::BoxStream;
|
||||
use lance::Dataset;
|
||||
use lance::io::WrappingObjectStore;
|
||||
use object_store::path::Path;
|
||||
use object_store::{
|
||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||
ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result,
|
||||
};
|
||||
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
use crate::metadata_index::{self, State};
|
||||
|
||||
/// Removes the table directory just before the first write through it, the
|
||||
/// way DeleteIndex purging the table between a job's open and its writes
|
||||
/// would.
|
||||
#[derive(Debug)]
|
||||
struct DropOnFirstWrite {
|
||||
dir: PathBuf,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for DropOnFirstWrite {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(Dropping {
|
||||
inner: original,
|
||||
dir: self.dir.clone(),
|
||||
fired: self.fired.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Dropping {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
dir: PathBuf,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Dropping {
|
||||
fn drop_once(&self) {
|
||||
if !self.fired.swap(true, Ordering::SeqCst) {
|
||||
std::fs::remove_dir_all(&self.dir).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Dropping {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "Dropping({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for Dropping {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
self.drop_once();
|
||||
self.inner.put_opts(location, payload, opts).await
|
||||
}
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
self.drop_once();
|
||||
self.inner.put_multipart_opts(location, opts).await
|
||||
}
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
self.inner.delete_stream(locations)
|
||||
}
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens the table the way a job does; `during` drops it at the job's first
|
||||
/// write, otherwise it is dropped right after the open.
|
||||
async fn open_then_drop(dir: &tempfile::TempDir, during: bool) -> Dataset {
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let dropping: Option<Arc<dyn WrappingObjectStore>> = during.then(|| {
|
||||
Arc::new(DropOnFirstWrite {
|
||||
dir: dir.path().to_path_buf(),
|
||||
fired: Arc::new(AtomicBool::new(false)),
|
||||
}) as Arc<dyn WrappingObjectStore>
|
||||
});
|
||||
let dataset = crate::dataset::open_guarded(&description, dropping)
|
||||
.await
|
||||
.unwrap();
|
||||
if !during {
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
}
|
||||
dataset
|
||||
}
|
||||
|
||||
fn files(dir: &std::path::Path) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if let Ok(entries) = std::fs::read_dir(dir) {
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
out.extend(files(&path));
|
||||
} else {
|
||||
out.push(path.display().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Runs `job` on a table dropped right after the job opens it, or at the
|
||||
/// job's first write. The job must fail, leave no manifest (nor, dropped
|
||||
/// before it wrote, any file) in the dropped path, and a table created
|
||||
/// there again must start empty.
|
||||
async fn dropped_during<F, Fut>(
|
||||
label: &str,
|
||||
during: bool,
|
||||
prepare: impl AsyncFnOnce(&mut Dataset),
|
||||
job: F,
|
||||
) where
|
||||
F: FnOnce(Dataset) -> Fut,
|
||||
Fut: std::future::Future<Output = anyhow::Result<()>>,
|
||||
{
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
prepare(&mut dataset).await;
|
||||
let result = job(open_then_drop(&dir, during).await).await;
|
||||
let left = files(dir.path());
|
||||
let error = result.expect_err(&format!(
|
||||
"{label}: a job on a dropped table succeeded, leaving {left:?}"
|
||||
));
|
||||
eprintln!(
|
||||
"{label} (dropped {}): {error:#}",
|
||||
if during {
|
||||
"at its first write"
|
||||
} else {
|
||||
"after the open"
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
!left.iter().any(|f| f.ends_with(".manifest")),
|
||||
"{label}: a manifest was written into the dropped table: {left:?}"
|
||||
);
|
||||
if !during {
|
||||
assert!(
|
||||
left.is_empty(),
|
||||
"{label}: files were written into the dropped table: {left:?}"
|
||||
);
|
||||
}
|
||||
let recreated = Dataset::write(
|
||||
arrow_array::RecordBatchIterator::new(
|
||||
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
|
||||
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
|
||||
),
|
||||
dir.path().to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("{label}: the table could not be created again: {e}"));
|
||||
assert_eq!(recreated.version().version, 1, "{label}");
|
||||
assert_eq!(recreated.count_rows(None).await.unwrap(), 0, "{label}");
|
||||
}
|
||||
|
||||
/// Every job, with the table dropped after its open and at its first write.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_job_on_a_dropped_table_fails_and_leaves_it_dropped() {
|
||||
for during in [false, true] {
|
||||
dropped_during(
|
||||
"vector index build",
|
||||
during,
|
||||
async |_| {},
|
||||
async |mut d| {
|
||||
crate::vector_index::build(
|
||||
&mut d,
|
||||
"vector",
|
||||
crate::vector_index::INDEX_NAME,
|
||||
4,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key add",
|
||||
during,
|
||||
async |d| metadata_index::request(d, "author").await.unwrap(),
|
||||
async |mut d| {
|
||||
let key = metadata_index::add_column(&mut d, "author").await?;
|
||||
metadata_index::finish_after(&mut d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.map(|_| ())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key maintenance",
|
||||
during,
|
||||
async |d| {
|
||||
let key = metadata_index::add_column(d, "author").await.unwrap();
|
||||
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
*d = upsert(d.clone(), 4_000..4_500).await;
|
||||
},
|
||||
async |mut d| {
|
||||
let key = metadata_index::keys_of(&d).remove(0);
|
||||
metadata_index::maintain(&mut d, &key, 1_000, 0)
|
||||
.await
|
||||
.map(|_| ())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dropped_during(
|
||||
"metadata key remove",
|
||||
during,
|
||||
async |d| {
|
||||
let key = metadata_index::add_column(d, "author").await.unwrap();
|
||||
metadata_index::finish_after(d, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::set_state(d, "_mk_0", State::Off)
|
||||
.await
|
||||
.unwrap();
|
||||
},
|
||||
async |mut d| metadata_index::remove_column(&mut d, "_mk_0").await,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A job that removes old versions itself, as a purge does, removes the
|
||||
/// manifest it opened: that is no drop, and its later writes go through.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_purge_through_the_guard_completes() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let dataset = table(&dir, 4).await;
|
||||
let mut dataset = upsert(dataset, 0..100).await;
|
||||
dataset.delete("key = 'k1' OR key = 'k2001'").await.unwrap();
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let mut guarded = crate::dataset::open_guarded(&description, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// The job commits, then removes every older version, the one it opened
|
||||
// with them.
|
||||
guarded
|
||||
.update_config([("seaweedfs.test", "1")])
|
||||
.await
|
||||
.unwrap();
|
||||
let removed = lance::dataset::cleanup::cleanup_old_versions(
|
||||
&guarded,
|
||||
lance::dataset::cleanup::CleanupPolicy {
|
||||
before_timestamp: Some(chrono::Utc::now() + chrono::Duration::hours(1)),
|
||||
before_version: Some(guarded.version().version),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(removed.old_versions > 0);
|
||||
guarded
|
||||
.update_config([("seaweedfs.test", "2")])
|
||||
.await
|
||||
.unwrap();
|
||||
crate::jobs::purge::purge(&mut guarded, "t1").await.unwrap();
|
||||
assert_eq!(guarded.count_rows(None).await.unwrap(), 3_998);
|
||||
}
|
||||
|
||||
/// The table is dropped and created again up to the version the job opened,
|
||||
/// so the manifest path the job watches exists again, as another object:
|
||||
/// the job must still fail and leave the new table as it was.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_job_on_a_table_created_again_fails() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
let key = metadata_index::add_column(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::finish_after(&mut dataset, &key, 1_000, std::time::Duration::ZERO)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::set_state(&mut dataset, "_mk_0", State::Off)
|
||||
.await
|
||||
.unwrap();
|
||||
let description = crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let mut job = crate::dataset::open_guarded(&description, None)
|
||||
.await
|
||||
.unwrap();
|
||||
let opened = job.version().version;
|
||||
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let mut recreated = table(&dir, 1).await;
|
||||
while recreated.version().version < opened {
|
||||
let version = recreated.version().version.to_string();
|
||||
recreated
|
||||
.update_config([("seaweedfs.test", version.as_str())])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(recreated.version().version, opened);
|
||||
|
||||
let error = metadata_index::remove_column(&mut job, "_mk_0")
|
||||
.await
|
||||
.expect_err("a job on a table created again succeeded");
|
||||
assert!(format!("{error:#}").contains("DropGuard"), "{error:#}");
|
||||
recreated.checkout_latest().await.unwrap();
|
||||
assert_eq!(recreated.version().version, opened);
|
||||
assert_eq!(recreated.count_rows(None).await.unwrap(), 1_000);
|
||||
}
|
||||
|
||||
fn guarded_description(dir: &tempfile::TempDir) -> crate::catalog::TableDescription {
|
||||
crate::catalog::TableDescription {
|
||||
location: dir.path().to_str().unwrap().to_string(),
|
||||
storage_options: Default::default(),
|
||||
managed_versioning: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The table's directory in the store's own paths.
|
||||
fn table_root(dataset: &Dataset) -> Path {
|
||||
let manifest = &dataset.manifest_location().path;
|
||||
manifest
|
||||
.parts()
|
||||
.take(manifest.parts().count() - 2)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A multipart upload started before the drop does not complete after it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_multipart_upload_does_not_complete_into_a_dropped_table() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like::default());
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
let path = table_root(&job).child("data").child("late.lance");
|
||||
let mut upload = store.put_multipart(&path).await.unwrap();
|
||||
upload
|
||||
.put_part(PutPayload::from(vec![0u8; 16]))
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let completed = upload.complete().await;
|
||||
assert!(
|
||||
completed.is_err(),
|
||||
"an upload completed into a dropped table"
|
||||
);
|
||||
assert!(files(dir.path()).is_empty(), "{:?}", files(dir.path()));
|
||||
}
|
||||
|
||||
/// An upload whose last probe fails is aborted, not left with its parts.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_probe_aborts_an_upload() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like = S3Like::default();
|
||||
let (armed, aborted) = (s3_like.armed.clone(), s3_like.aborted.clone());
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(s3_like);
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
let path = table_root(&job).child("data").child("late.lance");
|
||||
let mut upload = store.put_multipart(&path).await.unwrap();
|
||||
upload
|
||||
.put_part(PutPayload::from(vec![0u8; 16]))
|
||||
.await
|
||||
.unwrap();
|
||||
armed.store(true, Ordering::SeqCst);
|
||||
assert!(upload.complete().await.is_err());
|
||||
assert!(aborted.load(Ordering::SeqCst), "the upload was left open");
|
||||
}
|
||||
|
||||
/// A table created again, through the version a job just wrote, between
|
||||
/// that write and its check keeps its own manifest at that path.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_rollback_keeps_the_manifest_of_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(job.version().version, 1);
|
||||
assert!(
|
||||
job.update_config([("seaweedfs.test", "job")])
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(again.version().version, 2);
|
||||
assert_eq!(
|
||||
again.config().get("seaweedfs.test").map(String::as_str),
|
||||
Some("again")
|
||||
);
|
||||
}
|
||||
|
||||
/// A table created again whose manifest at the job's path cannot be told
|
||||
/// from the job's, because the HEAD fails, keeps it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_rollback_probe_keeps_the_manifest_of_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let s3_like: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
recreate: Some((dir.path().to_path_buf(), Arc::new(AtomicBool::new(false)))),
|
||||
fail_head_after_manifest: true,
|
||||
heads_before_failing: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(s3_like))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
job.update_config([("seaweedfs.test", "job")])
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let again = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(again.version().version, 2);
|
||||
assert_eq!(
|
||||
again.config().get("seaweedfs.test").map(String::as_str),
|
||||
Some("again")
|
||||
);
|
||||
}
|
||||
|
||||
/// A cleanup that picked its paths before the drop does not delete the files
|
||||
/// of the table created again there.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_cleanup_does_not_delete_a_table_created_again() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 1).await;
|
||||
dataset
|
||||
.update_config([("seaweedfs.test", "1")])
|
||||
.await
|
||||
.unwrap();
|
||||
let job = crate::dataset::open_guarded(&guarded_description(&dir), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let store = job.object_store(None).await.unwrap().inner.clone();
|
||||
// The old version-1 manifest, as a cleanup would have picked it.
|
||||
let mut first = dataset.checkout_version(1).await.unwrap();
|
||||
let old = first.manifest_location().path.clone();
|
||||
std::fs::remove_dir_all(dir.path()).unwrap();
|
||||
let recreated = table(&dir, 1).await;
|
||||
assert_eq!(recreated.manifest_location().path, old);
|
||||
let deleted: Vec<_> = store
|
||||
.delete_stream(futures::stream::iter(vec![Ok(old.clone())]).boxed())
|
||||
.collect()
|
||||
.await;
|
||||
assert!(deleted.iter().any(|r| r.is_err()), "{deleted:?}");
|
||||
first = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(first.version().version, 1);
|
||||
}
|
||||
|
||||
/// A probe that fails after a manifest was written says nothing about the
|
||||
/// table: the manifest stays and the commit succeeds.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_failed_probe_does_not_take_back_a_commit() {
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
table(&dir, 1).await;
|
||||
let flaky: Arc<dyn WrappingObjectStore> = Arc::new(S3Like {
|
||||
fail_head_after_manifest: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut job = crate::dataset::open_guarded(&guarded_description(&dir), Some(flaky))
|
||||
.await
|
||||
.unwrap();
|
||||
let before = job.version().version;
|
||||
job.update_config([("seaweedfs.test", "1")]).await.unwrap();
|
||||
let reopened = Dataset::open(dir.path().to_str().unwrap()).await.unwrap();
|
||||
assert_eq!(reopened.version().version, before + 1);
|
||||
}
|
||||
|
||||
/// Behaves as an object store, not a file system: a multipart upload holds
|
||||
/// its parts until it completes, and records an abort. Optionally fails one
|
||||
/// HEAD after a manifest is written, as a timeout would, or creates the table
|
||||
/// again, through two versions, right after the first manifest is written.
|
||||
#[derive(Debug, Default)]
|
||||
struct S3Like {
|
||||
fail_head_after_manifest: bool,
|
||||
/// HEADs let through after a manifest is written before one fails.
|
||||
heads_before_failing: usize,
|
||||
/// The next HEAD fails.
|
||||
armed: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
|
||||
}
|
||||
|
||||
impl WrappingObjectStore for S3Like {
|
||||
fn wrap(&self, _prefix: &str, original: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
|
||||
Arc::new(S3LikeStore {
|
||||
inner: original,
|
||||
fail_head_after_manifest: self.fail_head_after_manifest,
|
||||
heads_before_failing: self.heads_before_failing,
|
||||
heads_left: Arc::new(AtomicUsize::new(0)),
|
||||
armed: self.armed.clone(),
|
||||
aborted: self.aborted.clone(),
|
||||
recreate: self.recreate.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct S3LikeStore {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
fail_head_after_manifest: bool,
|
||||
heads_before_failing: usize,
|
||||
heads_left: Arc<AtomicUsize>,
|
||||
armed: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
recreate: Option<(PathBuf, Arc<AtomicBool>)>,
|
||||
}
|
||||
|
||||
impl fmt::Display for S3LikeStore {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "S3Like({})", self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectStore for S3LikeStore {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
opts: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
let put = self.inner.put_opts(location, payload, opts).await?;
|
||||
if location.extension() == Some("manifest") {
|
||||
if self.fail_head_after_manifest {
|
||||
self.heads_left
|
||||
.store(self.heads_before_failing, Ordering::SeqCst);
|
||||
self.armed.store(true, Ordering::SeqCst);
|
||||
}
|
||||
if let Some((dir, fired)) = &self.recreate
|
||||
&& !fired.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
let mut again = create_empty(dir).await;
|
||||
again
|
||||
.update_config([("seaweedfs.test", "again")])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
Ok(put)
|
||||
}
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
_opts: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
Ok(Box::new(HeldParts {
|
||||
inner: self.inner.clone(),
|
||||
location: location.clone(),
|
||||
parts: Vec::new(),
|
||||
aborted: self.aborted.clone(),
|
||||
}))
|
||||
}
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
if options.head
|
||||
&& self.armed.load(Ordering::SeqCst)
|
||||
&& self
|
||||
.heads_left
|
||||
.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
|
||||
.is_err()
|
||||
&& self.armed.swap(false, Ordering::SeqCst)
|
||||
{
|
||||
return Err(object_store::Error::Generic {
|
||||
store: "S3Like",
|
||||
source: "timed out".into(),
|
||||
});
|
||||
}
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
|
||||
self.inner.get_ranges(location, ranges).await
|
||||
}
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
self.inner.delete_stream(locations)
|
||||
}
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
fn list_with_offset(
|
||||
&self,
|
||||
prefix: Option<&Path>,
|
||||
offset: &Path,
|
||||
) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list_with_offset(prefix, offset)
|
||||
}
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
|
||||
self.inner.rename_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Parts held outside the table until the upload completes, as S3 does.
|
||||
#[derive(Debug)]
|
||||
struct HeldParts {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
location: Path,
|
||||
parts: Vec<Bytes>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MultipartUpload for HeldParts {
|
||||
fn put_part(&mut self, data: PutPayload) -> object_store::UploadPart {
|
||||
self.parts.extend(data);
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
async fn complete(&mut self) -> Result<PutResult> {
|
||||
let payload: PutPayload = self.parts.drain(..).collect();
|
||||
self.inner.put(&self.location, payload).await
|
||||
}
|
||||
async fn abort(&mut self) -> Result<()> {
|
||||
self.parts.clear();
|
||||
self.aborted.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// An empty table at `dir`.
|
||||
async fn create_empty(dir: &std::path::Path) -> Dataset {
|
||||
Dataset::write(
|
||||
arrow_array::RecordBatchIterator::new(
|
||||
Vec::<std::result::Result<arrow_array::RecordBatch, arrow_schema::ArrowError>>::new(),
|
||||
Arc::new(crate::metadata_index::tests::lifecycle::base_schema()),
|
||||
),
|
||||
dir.to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
@@ -0,0 +1,757 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use anyhow::{Result, anyhow, bail};
|
||||
use async_trait::async_trait;
|
||||
use seaweed_worker_core::config_form::{form, int_or, int_value, number_field, string_value};
|
||||
use seaweed_worker_core::pb::{
|
||||
AdminRuntimeDefaults, ConfigValue, DetectionComplete, DetectionProposals, ExecuteJobRequest,
|
||||
JobCompleted, JobProgressUpdate, JobProposal, JobResult, JobTypeCapability, JobTypeDescriptor,
|
||||
RunDetectionRequest, config_value::Kind,
|
||||
};
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::vector_index::{
|
||||
EXECUTION_TIMEOUT_SECONDS, declare_build_runtime, is_vector_table,
|
||||
};
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
use crate::metadata_index::{self, DEFAULT_MIN_UNINDEXED_ROWS, DEFAULT_SEGMENT_ROWS, State};
|
||||
|
||||
pub const JOB_TYPE: &str = "lance_metadata_index";
|
||||
|
||||
const SEGMENT_ROWS_FLOOR: i64 = 1_000;
|
||||
const SEGMENT_ROWS_CEILING: i64 = 1_000_000_000;
|
||||
const MIN_UNINDEXED_FLOOR: i64 = 0;
|
||||
const MIN_UNINDEXED_CEILING: i64 = 100_000_000;
|
||||
|
||||
/// What a job does to one key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Action {
|
||||
/// Add the requested key's column, or finish a `building` one.
|
||||
Add,
|
||||
/// Drop an `off` key's index and column.
|
||||
Remove,
|
||||
/// Fill NULL rows and fold unindexed fragments of a `ready` key.
|
||||
Maintain,
|
||||
}
|
||||
|
||||
impl Action {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Action::Add => "add",
|
||||
Action::Remove => "remove",
|
||||
Action::Maintain => "maintain",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"add" => Some(Action::Add),
|
||||
"remove" => Some(Action::Remove),
|
||||
"maintain" => Some(Action::Maintain),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The work an index's table asks for: requested and `building` keys to add,
|
||||
/// `off` keys to remove, and `ready` keys whose unindexed rows reached
|
||||
/// `min_unindexed_rows`.
|
||||
pub async fn pending(
|
||||
dataset: &lance::Dataset,
|
||||
min_unindexed_rows: u64,
|
||||
) -> Result<Vec<(String, Action)>> {
|
||||
let mut out: Vec<(String, Action)> = metadata_index::requests(dataset)
|
||||
.into_iter()
|
||||
.map(|key| (key, Action::Add))
|
||||
.collect();
|
||||
for key in metadata_index::keys_of(dataset) {
|
||||
let action = match key.state {
|
||||
Some(State::Building) => Action::Add,
|
||||
Some(State::Off) => Action::Remove,
|
||||
Some(State::Ready) => {
|
||||
let segments = metadata_index::segments(dataset, &key.column).await?;
|
||||
let pending = metadata_index::unindexed(dataset, &segments);
|
||||
let unindexed: u64 = pending.iter().map(|(_, rows)| rows).sum();
|
||||
if unindexed == 0 {
|
||||
continue;
|
||||
}
|
||||
// Rows a write left NULL (one that raced the key's add and
|
||||
// stopped before fixing them) wait for no threshold: queries
|
||||
// through the key miss them until they are derived again.
|
||||
// Nor does the first segment of a key that went ready on an
|
||||
// empty table.
|
||||
if unindexed < min_unindexed_rows
|
||||
&& !segments.is_empty()
|
||||
&& !has_nulls(dataset, &key.column, &pending).await?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Action::Maintain
|
||||
}
|
||||
None => continue,
|
||||
};
|
||||
out.push((key.key, action));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Whether `fragments` hold a row whose `column` is NULL.
|
||||
async fn has_nulls(
|
||||
dataset: &lance::Dataset,
|
||||
column: &str,
|
||||
fragments: &[(u32, u64)],
|
||||
) -> Result<bool> {
|
||||
let ids: Vec<u64> = fragments.iter().map(|(id, _)| *id as u64).collect();
|
||||
let mut scanner = dataset.scan();
|
||||
scanner
|
||||
.filter(&format!("`{}` IS NULL", column.replace('`', "``")))?
|
||||
.with_fragments(
|
||||
dataset
|
||||
.fragments()
|
||||
.iter()
|
||||
.filter(|f| ids.contains(&f.id))
|
||||
.cloned()
|
||||
.collect(),
|
||||
);
|
||||
Ok(scanner.count_rows().await? > 0)
|
||||
}
|
||||
|
||||
/// Carries out `action` on `key`, as the table now records it; an action the
|
||||
/// table no longer asks for is skipped. Returns what was done.
|
||||
pub async fn run(
|
||||
dataset: &mut lance::Dataset,
|
||||
key: &str,
|
||||
action: Action,
|
||||
segment_rows: u64,
|
||||
min_unindexed_rows: u64,
|
||||
grace: std::time::Duration,
|
||||
) -> Result<String> {
|
||||
let column = metadata_index::keys_of(dataset)
|
||||
.into_iter()
|
||||
.find(|k| k.key == key);
|
||||
match (action, column) {
|
||||
(Action::Add, None) => {
|
||||
if !metadata_index::requests(dataset).iter().any(|k| k == key) {
|
||||
return Ok(format!("{key} is no longer requested"));
|
||||
}
|
||||
let added = metadata_index::add_column(dataset, key).await?;
|
||||
if added.state != Some(State::Building) {
|
||||
return Ok(format!(
|
||||
"{key} was withdrawn while {} was added",
|
||||
added.column
|
||||
));
|
||||
}
|
||||
let filled = metadata_index::finish_after(dataset, &added, segment_rows, grace).await?;
|
||||
Ok(format!(
|
||||
"{key} indexed in {} ({filled} rows a racing write left derived again)",
|
||||
added.column
|
||||
))
|
||||
}
|
||||
(Action::Add, Some(column)) if column.state == Some(State::Building) => {
|
||||
// An add that stopped before setting again what its commit
|
||||
// overwrote leaves that to the build resuming it.
|
||||
let state = metadata_index::recover_add(dataset, &column).await?;
|
||||
if state != Some(State::Building) {
|
||||
return Ok(format!(
|
||||
"{key} is {} after its add was recovered",
|
||||
state.map_or("unknown", State::as_str)
|
||||
));
|
||||
}
|
||||
let filled =
|
||||
metadata_index::finish_after(dataset, &column, segment_rows, grace).await?;
|
||||
Ok(format!(
|
||||
"{key} indexed in {} ({filled} rows a racing write left derived again)",
|
||||
column.column
|
||||
))
|
||||
}
|
||||
(Action::Remove, Some(column)) if column.state == Some(State::Off) => {
|
||||
// Dropping the column drops its restore mark: set again first
|
||||
// what its add overwrote.
|
||||
metadata_index::recover_add(dataset, &column).await?;
|
||||
metadata_index::remove_column(dataset, &column.column).await?;
|
||||
Ok(format!("{key} removed with {}", column.column))
|
||||
}
|
||||
(Action::Maintain, Some(column)) if column.state == Some(State::Ready) => {
|
||||
let (filled, folded) =
|
||||
metadata_index::maintain(dataset, &column, segment_rows, min_unindexed_rows)
|
||||
.await?;
|
||||
Ok(format!("{key}: {folded:?}, {filled} rows derived again"))
|
||||
}
|
||||
(action, column) => Ok(format!(
|
||||
"{key} needs no {}: {}",
|
||||
action.as_str(),
|
||||
column
|
||||
.and_then(|c| c.state)
|
||||
.map_or("not indexed", State::as_str)
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Adds, maintains and removes the indexed metadata keys of S3 Vectors
|
||||
/// indexes, as each index's table records them. The admin dispatches a job
|
||||
/// when an operator asks; detection finds the rest.
|
||||
pub struct MetadataIndexHandler {
|
||||
namespace_url: String,
|
||||
storage: dataset::Storage,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
impl MetadataIndexHandler {
|
||||
pub fn new(namespace_url: String) -> Self {
|
||||
Self {
|
||||
namespace_url,
|
||||
storage: dataset::Storage::default(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_metrics(mut self, metrics: Option<crate::metrics::LanceMetrics>) -> Self {
|
||||
self.metrics = metrics;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_storage(mut self, storage: dataset::Storage) -> Self {
|
||||
self.storage = storage;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn settings(values: &HashMap<String, ConfigValue>) -> (u64, u64) {
|
||||
(
|
||||
clamp(
|
||||
int_or(values, "segment_rows", DEFAULT_SEGMENT_ROWS as i64),
|
||||
SEGMENT_ROWS_FLOOR,
|
||||
SEGMENT_ROWS_CEILING,
|
||||
) as u64,
|
||||
clamp(
|
||||
int_or(
|
||||
values,
|
||||
"min_unindexed_rows",
|
||||
DEFAULT_MIN_UNINDEXED_ROWS as i64,
|
||||
),
|
||||
MIN_UNINDEXED_FLOOR,
|
||||
MIN_UNINDEXED_CEILING,
|
||||
) as u64,
|
||||
)
|
||||
}
|
||||
|
||||
fn string_param(parameters: &HashMap<String, ConfigValue>, name: &str) -> Option<String> {
|
||||
match parameters.get(name)?.kind.as_ref()? {
|
||||
Kind::StringValue(s) => Some(s.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for MetadataIndexHandler {
|
||||
fn capability(&self) -> JobTypeCapability {
|
||||
JobTypeCapability {
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
can_detect: true,
|
||||
can_execute: true,
|
||||
max_detection_concurrency: 1,
|
||||
max_execution_concurrency: 1,
|
||||
display_name: "Lance Metadata Index".to_string(),
|
||||
description: "Add, maintain and remove the indexed metadata keys of S3 Vectors indexes"
|
||||
.to_string(),
|
||||
weight: 40,
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor(&self) -> JobTypeDescriptor {
|
||||
let mut defaults: HashMap<String, ConfigValue> = HashMap::new();
|
||||
defaults.insert(
|
||||
"segment_rows".to_string(),
|
||||
int_value(DEFAULT_SEGMENT_ROWS as i64),
|
||||
);
|
||||
defaults.insert(
|
||||
"min_unindexed_rows".to_string(),
|
||||
int_value(DEFAULT_MIN_UNINDEXED_ROWS as i64),
|
||||
);
|
||||
JobTypeDescriptor {
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
display_name: "Lance Metadata Index".to_string(),
|
||||
description:
|
||||
"Keep a LABEL_LIST index for each metadata key an S3 Vectors index asks for"
|
||||
.to_string(),
|
||||
icon: "fas fa-tags".to_string(),
|
||||
descriptor_version: 1,
|
||||
worker_config_form: Some(form(
|
||||
"lance-metadata-index-worker",
|
||||
"Metadata index",
|
||||
vec![
|
||||
number_field(
|
||||
"segment_rows",
|
||||
"Segment rows",
|
||||
"Rows an index segment takes before a new one starts",
|
||||
SEGMENT_ROWS_FLOOR,
|
||||
SEGMENT_ROWS_CEILING,
|
||||
),
|
||||
number_field(
|
||||
"min_unindexed_rows",
|
||||
"Unindexed rows",
|
||||
"Rows written since the last fold that wait for the next one",
|
||||
MIN_UNINDEXED_FLOOR,
|
||||
MIN_UNINDEXED_CEILING,
|
||||
),
|
||||
],
|
||||
defaults.clone(),
|
||||
)),
|
||||
worker_default_values: defaults,
|
||||
admin_runtime_defaults: Some(AdminRuntimeDefaults {
|
||||
job_type_max_runtime_seconds: EXECUTION_TIMEOUT_SECONDS,
|
||||
execution_timeout_seconds: EXECUTION_TIMEOUT_SECONDS,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn detect(
|
||||
&self,
|
||||
request: &RunDetectionRequest,
|
||||
sender: &dyn DetectionSender,
|
||||
) -> Result<()> {
|
||||
let (_, min_unindexed_rows) = settings(&request.worker_config_values);
|
||||
let max_results = usize::try_from(request.max_results).unwrap_or(0);
|
||||
let client = self.client();
|
||||
let mut proposals = Vec::new();
|
||||
let mut has_more = false;
|
||||
'tables: for encoded in &client.list_all_tables().await? {
|
||||
let id = parse_id(encoded);
|
||||
if !is_vector_table(&id) {
|
||||
continue;
|
||||
}
|
||||
let table = match dataset::open(&client, &id, &self.storage).await {
|
||||
Ok(table) => table,
|
||||
Err(err) => {
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.worker.object_skipped(JOB_TYPE, "open");
|
||||
}
|
||||
warn!("skipping {encoded}: {err:#}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(counters) = &self.metrics {
|
||||
counters.worker.object_seen(JOB_TYPE);
|
||||
}
|
||||
let work = match pending(&table.dataset, min_unindexed_rows).await {
|
||||
Ok(work) => work,
|
||||
Err(err) => {
|
||||
warn!("skipping {encoded}: reading its indexed keys failed: {err:#}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for (key, action) in work {
|
||||
if max_results > 0 && proposals.len() >= max_results {
|
||||
has_more = true;
|
||||
break 'tables;
|
||||
}
|
||||
proposals.push(proposal(&id, encoded, &key, action));
|
||||
}
|
||||
}
|
||||
let total = proposals.len() as i32;
|
||||
sender.send_proposals(DetectionProposals {
|
||||
request_id: request.request_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
proposals,
|
||||
has_more,
|
||||
})?;
|
||||
sender.send_complete(DetectionComplete {
|
||||
request_id: request.request_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
error_message: String::new(),
|
||||
total_proposals: total,
|
||||
skipped_reason: String::new(),
|
||||
eligible_after: None,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
request: &ExecuteJobRequest,
|
||||
sender: &dyn ExecutionSender,
|
||||
) -> Result<()> {
|
||||
let job = request
|
||||
.job
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow!("execute request carried no job"))?;
|
||||
let id = table_id(&job.parameters)
|
||||
.ok_or_else(|| anyhow!("job {} carried no table_id", job.job_id))?;
|
||||
if !is_vector_table(&id) {
|
||||
bail!("{} is not an S3 Vectors index", id.join("/"));
|
||||
}
|
||||
let key = string_param(&job.parameters, "key")
|
||||
.ok_or_else(|| anyhow!("job {} carried no key", job.job_id))?;
|
||||
let action = string_param(&job.parameters, "action")
|
||||
.as_deref()
|
||||
.and_then(Action::parse)
|
||||
.ok_or_else(|| anyhow!("job {} carried no valid action", job.job_id))?;
|
||||
let (segment_rows, min_unindexed_rows) = settings(&request.worker_config_values);
|
||||
sender.send_progress(JobProgressUpdate {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
progress_percent: 10.0,
|
||||
stage: format!("{} {key}", action.as_str()),
|
||||
..Default::default()
|
||||
})?;
|
||||
let client = self.client();
|
||||
let mut table = dataset::open(&client, &id, &self.storage).await?;
|
||||
let summary = run(
|
||||
&mut table.dataset,
|
||||
&key,
|
||||
action,
|
||||
segment_rows,
|
||||
min_unindexed_rows,
|
||||
metadata_index::RACE_GRACE,
|
||||
)
|
||||
.await?;
|
||||
sender.send_completed(JobCompleted {
|
||||
request_id: request.request_id.clone(),
|
||||
job_id: job.job_id.clone(),
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
success: true,
|
||||
result: Some(JobResult {
|
||||
summary,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// The proposal for one key of one table; the admin builds the same
|
||||
/// parameters when it dispatches a job for an operator.
|
||||
fn proposal(id: &[String], encoded: &str, key: &str, action: Action) -> JobProposal {
|
||||
let mut parameters: HashMap<String, ConfigValue> = HashMap::new();
|
||||
parameters.insert("table_id".to_string(), string_list(id));
|
||||
parameters.insert("key".to_string(), string_value(key));
|
||||
parameters.insert("action".to_string(), string_value(action.as_str()));
|
||||
declare_build_runtime(&mut parameters);
|
||||
let dedupe = format!("{JOB_TYPE}:{encoded}:{key}:{}", action.as_str());
|
||||
JobProposal {
|
||||
proposal_id: dedupe.clone(),
|
||||
dedupe_key: dedupe,
|
||||
job_type: JOB_TYPE.to_string(),
|
||||
summary: format!("{} metadata key {key} of {encoded}", action.as_str()),
|
||||
parameters,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_proposal_names_the_table_key_and_action() {
|
||||
let id = parse_id("media$vector$movies");
|
||||
let p = proposal(&id, "media$vector$movies", "author", Action::Remove);
|
||||
assert_eq!(table_id(&p.parameters), Some(id));
|
||||
assert_eq!(
|
||||
string_param(&p.parameters, "key").as_deref(),
|
||||
Some("author")
|
||||
);
|
||||
assert_eq!(
|
||||
string_param(&p.parameters, "action")
|
||||
.as_deref()
|
||||
.and_then(Action::parse),
|
||||
Some(Action::Remove)
|
||||
);
|
||||
assert_eq!(
|
||||
p.dedupe_key,
|
||||
"lance_metadata_index:media$vector$movies:author:remove"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_are_held_to_their_ranges() {
|
||||
let mut values = HashMap::new();
|
||||
assert_eq!(
|
||||
settings(&values),
|
||||
(DEFAULT_SEGMENT_ROWS, DEFAULT_MIN_UNINDEXED_ROWS)
|
||||
);
|
||||
values.insert("segment_rows".to_string(), int_value(-5));
|
||||
values.insert("min_unindexed_rows".to_string(), int_value(-1));
|
||||
assert_eq!(settings(&values), (SEGMENT_ROWS_FLOOR as u64, 0));
|
||||
}
|
||||
|
||||
/// A removal of a key whose add stopped before setting again what it
|
||||
/// overwrote sets that first: a key requested meanwhile stays requested.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_removal_keeps_what_the_add_overwrote() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut adder = dataset.clone();
|
||||
metadata_index::request(&mut dataset, "lang").await.unwrap();
|
||||
metadata_index::commit_column(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::request_removal(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
run(
|
||||
&mut adder,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::keys_of(&adder).is_empty());
|
||||
assert_eq!(metadata_index::requests(&adder), ["lang"]);
|
||||
}
|
||||
|
||||
/// A withdrawal of a request an interrupted add overwrote sets that
|
||||
/// first, so recovering the add does not bring the request back.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_withdrawal_of_a_request_an_add_overwrote_stays_withdrawn() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut adder = dataset.clone();
|
||||
metadata_index::request(&mut dataset, "lang").await.unwrap();
|
||||
metadata_index::commit_column(&mut adder, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
metadata_index::request_removal(&mut dataset, "lang")
|
||||
.await
|
||||
.unwrap();
|
||||
let author = metadata_index::keys_of(&dataset).remove(0);
|
||||
metadata_index::recover_add(&mut dataset, &author)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::requests(&dataset).is_empty());
|
||||
}
|
||||
|
||||
/// A withdrawal that read the request before the key's column was added
|
||||
/// and committed after sets the column off.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_withdrawal_racing_the_add_sets_the_column_off() {
|
||||
use crate::metadata_index::tests::lifecycle::table;
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 2).await;
|
||||
metadata_index::request(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut operator = dataset.clone();
|
||||
let added = metadata_index::add_column(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(added.state, Some(State::Building));
|
||||
metadata_index::request_removal(&mut operator, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
dataset.checkout_latest().await.unwrap();
|
||||
assert_eq!(metadata_index::keys_of(&dataset)[0].state, Some(State::Off));
|
||||
assert!(metadata_index::requests(&dataset).is_empty());
|
||||
}
|
||||
|
||||
/// A key that went ready on an empty table builds its first segment
|
||||
/// once rows arrive, below the threshold.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn a_key_ready_without_a_segment_is_maintained_at_once() {
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 1).await;
|
||||
dataset.delete("true").await.unwrap();
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
metadata_index::keys_of(&dataset)[0].state,
|
||||
Some(State::Ready)
|
||||
);
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
|
||||
let mut dataset = upsert(dataset, 0..100).await;
|
||||
assert_eq!(
|
||||
pending(&dataset, 1_000_000).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
1_000_000,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
metadata_index::segments(&dataset, "_mk_0")
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// What the table asks for is what a job does, and a job the table no
|
||||
/// longer asks for does nothing.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn jobs_follow_what_the_table_records() {
|
||||
use crate::metadata_index::tests::lifecycle::{table, upsert};
|
||||
let dir = tempfile::TempDir::new().unwrap();
|
||||
let mut dataset = table(&dir, 4).await;
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
metadata_index::request(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_index::request(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pending(&dataset, 0).await.unwrap(),
|
||||
[
|
||||
("author".to_string(), Action::Add),
|
||||
("genre".to_string(), Action::Add)
|
||||
]
|
||||
);
|
||||
metadata_index::request_removal(&mut dataset, "genre")
|
||||
.await
|
||||
.unwrap();
|
||||
let skipped = run(
|
||||
&mut dataset,
|
||||
"genre",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(skipped, "genre is no longer requested");
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Add,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = metadata_index::keys_of(&dataset);
|
||||
assert_eq!(keys.len(), 1);
|
||||
assert_eq!(keys[0].state, Some(State::Ready));
|
||||
assert_eq!(
|
||||
metadata_index::segments(&dataset, "_mk_0")
|
||||
.await
|
||||
.unwrap()
|
||||
.len(),
|
||||
4
|
||||
);
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
|
||||
let mut dataset = upsert(dataset, 4_000..4_500).await;
|
||||
assert!(pending(&dataset, 1_000).await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
pending(&dataset, 500).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
500,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
|
||||
// A raced write that left NULL rows is maintained at once.
|
||||
let stale = dataset.clone();
|
||||
let mut dataset =
|
||||
crate::metadata_index::tests::lifecycle::upsert_with_null_columns(stale, 4_500..4_520)
|
||||
.await;
|
||||
assert_eq!(
|
||||
pending(&dataset, 1_000_000).await.unwrap(),
|
||||
[("author".to_string(), Action::Maintain)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Maintain,
|
||||
1_000,
|
||||
1_000_000,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(pending(&dataset, 1_000_000).await.unwrap().is_empty());
|
||||
let noop = run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(noop, "author needs no remove: ready");
|
||||
|
||||
metadata_index::request_removal(&mut dataset, "author")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
pending(&dataset, 0).await.unwrap(),
|
||||
[("author".to_string(), Action::Remove)]
|
||||
);
|
||||
run(
|
||||
&mut dataset,
|
||||
"author",
|
||||
Action::Remove,
|
||||
1_000,
|
||||
0,
|
||||
std::time::Duration::ZERO,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(metadata_index::keys_of(&dataset).is_empty());
|
||||
assert!(pending(&dataset, 0).await.unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
pub mod cleanup;
|
||||
pub mod compact;
|
||||
pub mod indices;
|
||||
pub mod metadata_index;
|
||||
pub mod purge;
|
||||
pub mod sort;
|
||||
pub mod vector_index;
|
||||
@@ -66,6 +67,12 @@ pub fn handlers(
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
metadata_index::MetadataIndexHandler::new(namespace_url.clone())
|
||||
.with_storage(storage.clone())
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
sort::SortHandler::new(namespace_url.clone())
|
||||
.with_storage(storage.clone())
|
||||
|
||||
@@ -5,10 +5,15 @@
|
||||
//! and its old versions removed. None can run in the Go worker, because all
|
||||
//! three read and rewrite Lance files. This crate is the worker that can.
|
||||
|
||||
// A job awaiting several Lance commits in turn needs deeper trait recursion
|
||||
// than the default to prove its future Send.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
pub mod catalog;
|
||||
pub mod cluster;
|
||||
pub mod dataset;
|
||||
pub mod direct_read;
|
||||
pub mod drop_guard;
|
||||
pub mod grpc_tls;
|
||||
pub mod jobs;
|
||||
pub mod metadata_index;
|
||||
@@ -31,3 +36,5 @@ pub mod volume_server_pb {
|
||||
pub mod remote_pb {
|
||||
tonic::include_proto!("remote_pb");
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod drop_tests;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type S3VectorsBucketsData struct {
|
||||
@@ -36,6 +37,7 @@ type S3VectorsBucketData struct {
|
||||
Bucket S3VectorsBucketSummary `json:"bucket"`
|
||||
Policy string `json:"policy,omitempty"`
|
||||
Indexes []*s3vectors.Index `json:"indexes"`
|
||||
Panels []S3VectorsIndexPanel `json:"panels"`
|
||||
Operations VectorOperationsData `json:"operations"`
|
||||
TotalIndexes int `json:"total_indexes"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
@@ -159,6 +161,16 @@ func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string)
|
||||
}
|
||||
data.Operations = s.GetVectorOperationsData(ctx)
|
||||
data.TotalIndexes = len(data.Indexes)
|
||||
data.Panels = make([]S3VectorsIndexPanel, len(data.Indexes))
|
||||
var panels errgroup.Group
|
||||
panels.SetLimit(8)
|
||||
for i, index := range data.Indexes {
|
||||
panels.Go(func() error {
|
||||
data.Panels[i] = s.s3VectorsIndexPanel(ctx, bucket, index, data.Operations.Engines)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = panels.Wait()
|
||||
return data, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// metadataIndexJobType is the worker job that adds, maintains and removes
|
||||
// indexed metadata keys.
|
||||
const metadataIndexJobType = "lance_metadata_index"
|
||||
|
||||
// frequentFilterKeys caps the "frequently filtered, not indexed" list.
|
||||
const frequentFilterKeys = 10
|
||||
|
||||
// S3VectorsIndexPanel is one index's card on the bucket page: its vector
|
||||
// count and vector index, its indexed metadata keys, and the keys its
|
||||
// queries filter on most that are not indexed.
|
||||
type S3VectorsIndexPanel struct {
|
||||
Index *s3vectors.Index `json:"index"`
|
||||
Vectors uint64 `json:"vectors"`
|
||||
VectorIndex string `json:"vector_index"`
|
||||
Keys []S3VectorsMetadataKeyRow `json:"keys"`
|
||||
Frequent []S3VectorsFilterKeyRow `json:"frequent"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StatsError string `json:"stats_error,omitempty"`
|
||||
// StatsTruncated marks stats from an engine that stopped counting new
|
||||
// keys, so Frequent can miss keys.
|
||||
StatsTruncated bool `json:"stats_truncated,omitempty"`
|
||||
stats map[string]*filterKeyTotal `json:"-"`
|
||||
}
|
||||
|
||||
type S3VectorsMetadataKeyRow struct {
|
||||
Key string `json:"key"`
|
||||
State string `json:"state"`
|
||||
Coverage float64 `json:"coverage"`
|
||||
QueriesPerHour float64 `json:"queries_per_hour"`
|
||||
IndexBytes uint64 `json:"index_bytes"`
|
||||
}
|
||||
|
||||
type S3VectorsFilterKeyRow struct {
|
||||
Key string `json:"key"`
|
||||
QueriesPerHour float64 `json:"queries_per_hour"`
|
||||
AvgMs float64 `json:"avg_ms"`
|
||||
// Numeric marks a key filtered only with numbers, which phase 1 does
|
||||
// not index.
|
||||
Numeric bool `json:"numeric"`
|
||||
}
|
||||
|
||||
type filterKeyTotal struct {
|
||||
perHour float64
|
||||
seconds float64
|
||||
queries uint64
|
||||
strings bool
|
||||
numbers bool
|
||||
}
|
||||
|
||||
// s3VectorsIndexPanel reads one index's keys from its table through an
|
||||
// engine, and its filter stats from every registered engine.
|
||||
func (s *AdminServer) s3VectorsIndexPanel(ctx context.Context, bucket string, index *s3vectors.Index, engines []VectorNode) S3VectorsIndexPanel {
|
||||
panel := S3VectorsIndexPanel{Index: index, Keys: []S3VectorsMetadataKeyRow{}, Frequent: []S3VectorsFilterKeyRow{}}
|
||||
keysCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
keys, err := s.s3VectorsServer.MetadataKeys(keysCtx, bucket, index.IndexName)
|
||||
if err != nil {
|
||||
panel.Error = err.Error()
|
||||
}
|
||||
stats, truncated, err := s.s3VectorsFilterStats(ctx, bucket, index.IndexName, engines)
|
||||
panel.StatsTruncated = truncated
|
||||
if err != nil {
|
||||
panel.StatsError = err.Error()
|
||||
}
|
||||
panel.stats = stats
|
||||
indexed := map[string]bool{}
|
||||
if keys != nil {
|
||||
panel.Vectors = keys.Rows
|
||||
panel.VectorIndex = keys.VectorIndex
|
||||
for _, k := range keys.Keys {
|
||||
indexed[k.Key] = true
|
||||
row := S3VectorsMetadataKeyRow{Key: k.Key, State: k.State, IndexBytes: k.IndexBytes}
|
||||
if k.Rows > 0 {
|
||||
row.Coverage = float64(k.IndexedRows) * 100 / float64(k.Rows)
|
||||
}
|
||||
if total := stats[k.Key]; total != nil {
|
||||
row.QueriesPerHour = total.perHour
|
||||
}
|
||||
panel.Keys = append(panel.Keys, row)
|
||||
}
|
||||
}
|
||||
nonFilterable := map[string]bool{}
|
||||
if index.MetadataConfiguration != nil {
|
||||
for _, k := range index.MetadataConfiguration.NonFilterableMetadataKeys {
|
||||
nonFilterable[k] = true
|
||||
}
|
||||
}
|
||||
for key, total := range stats {
|
||||
if indexed[key] || nonFilterable[key] || (!total.strings && !total.numbers) {
|
||||
continue
|
||||
}
|
||||
row := S3VectorsFilterKeyRow{Key: key, QueriesPerHour: total.perHour, Numeric: !total.strings}
|
||||
if total.queries > 0 {
|
||||
row.AvgMs = total.seconds * 1000 / float64(total.queries)
|
||||
}
|
||||
panel.Frequent = append(panel.Frequent, row)
|
||||
}
|
||||
sort.Slice(panel.Frequent, func(i, j int) bool {
|
||||
if panel.Frequent[i].QueriesPerHour != panel.Frequent[j].QueriesPerHour {
|
||||
return panel.Frequent[i].QueriesPerHour > panel.Frequent[j].QueriesPerHour
|
||||
}
|
||||
return panel.Frequent[i].Key < panel.Frequent[j].Key
|
||||
})
|
||||
if len(panel.Frequent) > frequentFilterKeys {
|
||||
panel.Frequent = panel.Frequent[:frequentFilterKeys]
|
||||
}
|
||||
return panel
|
||||
}
|
||||
|
||||
// s3VectorsFilterStats sums an index's filter stats over the engines, each
|
||||
// counted per hour over its own window.
|
||||
func (s *AdminServer) s3VectorsFilterStats(ctx context.Context, bucket, index string, engines []VectorNode) (map[string]*filterKeyTotal, bool, error) {
|
||||
responses := make([]*vector_pb.FilterStatsResponse, len(engines))
|
||||
errs := make([]error, len(engines))
|
||||
var calls errgroup.Group
|
||||
calls.SetLimit(8)
|
||||
for i, engine := range engines {
|
||||
calls.Go(func() error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
conn, err := pb.GrpcDial(ctx, engine.Address, false, s.grpcDialOption)
|
||||
if err != nil {
|
||||
errs[i] = err
|
||||
return nil
|
||||
}
|
||||
defer conn.Close()
|
||||
responses[i], errs[i] = vector_pb.NewVectorServiceClient(conn).FilterStats(ctx, &vector_pb.FilterStatsRequest{
|
||||
Table: &vector_pb.Table{Id: []string{bucket, "vector", index}},
|
||||
})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = calls.Wait()
|
||||
stats, truncated := sumFilterStats(responses)
|
||||
return stats, truncated, firstError(engines, errs)
|
||||
}
|
||||
|
||||
func sumFilterStats(responses []*vector_pb.FilterStatsResponse) (map[string]*filterKeyTotal, bool) {
|
||||
out := map[string]*filterKeyTotal{}
|
||||
truncated := false
|
||||
for _, resp := range responses {
|
||||
if resp == nil || resp.WindowSeconds == 0 {
|
||||
continue
|
||||
}
|
||||
truncated = truncated || resp.Truncated
|
||||
for _, k := range resp.Keys {
|
||||
total := out[k.Key]
|
||||
if total == nil {
|
||||
total = &filterKeyTotal{}
|
||||
out[k.Key] = total
|
||||
}
|
||||
total.perHour += float64(k.Queries) * 3600 / float64(resp.WindowSeconds)
|
||||
total.queries += k.Queries
|
||||
total.seconds += k.Seconds
|
||||
total.strings = total.strings || k.StringOperand
|
||||
total.numbers = total.numbers || k.NumberOperand
|
||||
}
|
||||
}
|
||||
return out, truncated
|
||||
}
|
||||
|
||||
func firstError(engines []VectorNode, errs []error) error {
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
return fmt.Errorf("filter stats from %s: %w", engines[i].Address, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateS3VectorsMetadataKeyAPI asks to index a metadata key (POST) or to
|
||||
// stop (DELETE), then dispatches the worker job that carries it out.
|
||||
func (s *AdminServer) UpdateS3VectorsMetadataKeyAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Index string `json:"index"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
if r.Method == http.MethodDelete {
|
||||
q := r.URL.Query()
|
||||
req.Bucket, req.Index, req.Key = q.Get("bucket"), q.Get("index"), q.Get("key")
|
||||
} else if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Bucket == "" || req.Index == "" || req.Key == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket, index and key are required")
|
||||
return
|
||||
}
|
||||
remove := r.Method == http.MethodDelete
|
||||
keys, err := s.s3VectorsServer.UpdateMetadataKey(r.Context(), req.Bucket, req.Index, req.Key, remove)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
// A withdrawn request leaves nothing for a worker to do.
|
||||
action := "add"
|
||||
if remove {
|
||||
action = ""
|
||||
for _, k := range keys.GetKeys() {
|
||||
if k.Key == req.Key && k.State == "off" {
|
||||
action = "remove"
|
||||
}
|
||||
}
|
||||
}
|
||||
dispatched := false
|
||||
if action != "" {
|
||||
dispatched = s.dispatchMetadataIndexJob(req.Bucket, req.Index, req.Key, action)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"keys": keys.GetKeys(), "job_dispatched": dispatched})
|
||||
}
|
||||
|
||||
// dispatchMetadataIndexJob runs the job for one key in the background when
|
||||
// a worker can; otherwise the job's detection finds the work.
|
||||
func (s *AdminServer) dispatchMetadataIndexJob(bucket, index, key, action string) bool {
|
||||
if s.plugin == nil || !s.hasCapableWorker(metadataIndexJobType) {
|
||||
return false
|
||||
}
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: uuid.NewString(),
|
||||
JobType: metadataIndexJobType,
|
||||
DedupeKey: fmt.Sprintf("%s:%s$vector$%s:%s:%s", metadataIndexJobType, bucket, index, key, action),
|
||||
Summary: fmt.Sprintf("%s metadata key %s of %s/%s", action, key, bucket, index),
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"table_id": {Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: []string{bucket, "vector", index}}}},
|
||||
"key": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: key}},
|
||||
"action": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: action}},
|
||||
},
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
|
||||
defer cancel()
|
||||
if _, err := s.plugin.ExecuteJob(ctx, job, s.buildDefaultPluginClusterContext(), 1); err != nil {
|
||||
glog.Warningf("%s for %s/%s key %s: %v", metadataIndexJobType, bucket, index, key, err)
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *AdminServer) hasCapableWorker(jobType string) bool {
|
||||
for _, worker := range s.GetPluginWorkers() {
|
||||
if capability := worker.Capabilities[jobType]; capability != nil && capability.CanExecute {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
||||
)
|
||||
|
||||
func TestSumFilterStatsPerHourAcrossEngines(t *testing.T) {
|
||||
got, truncated := sumFilterStats([]*vector_pb.FilterStatsResponse{
|
||||
{WindowSeconds: 3600, Keys: []*vector_pb.FilterKeyStats{
|
||||
{Key: "author", Queries: 100, Seconds: 2, StringOperand: true},
|
||||
{Key: "year", Queries: 10, Seconds: 0.5, NumberOperand: true},
|
||||
}},
|
||||
// An engine up for half an hour counts double per hour.
|
||||
{WindowSeconds: 1800, Keys: []*vector_pb.FilterKeyStats{
|
||||
{Key: "author", Queries: 50, Seconds: 1, StringOperand: true},
|
||||
}},
|
||||
nil,
|
||||
{WindowSeconds: 0, Keys: []*vector_pb.FilterKeyStats{{Key: "ignored", Queries: 5}}},
|
||||
})
|
||||
author := got["author"]
|
||||
if author == nil || math.Abs(author.perHour-200) > 1e-9 || author.queries != 150 || math.Abs(author.seconds-3) > 1e-9 || !author.strings {
|
||||
t.Fatalf("author: %+v", author)
|
||||
}
|
||||
if year := got["year"]; year == nil || year.strings || !year.numbers || year.perHour != 10 {
|
||||
t.Fatalf("year: %+v", year)
|
||||
}
|
||||
if _, ok := got["ignored"]; ok {
|
||||
t.Fatal("an engine with no window must not count")
|
||||
}
|
||||
if truncated {
|
||||
t.Fatal("no engine stopped counting keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSumFilterStatsKeepsTruncated(t *testing.T) {
|
||||
_, truncated := sumFilterStats([]*vector_pb.FilterStatsResponse{
|
||||
{WindowSeconds: 3600, Keys: []*vector_pb.FilterKeyStats{{Key: "author", Queries: 1}}},
|
||||
{WindowSeconds: 3600, Truncated: true},
|
||||
})
|
||||
if !truncated {
|
||||
t.Fatal("one engine's truncated stats make the sum truncated")
|
||||
}
|
||||
}
|
||||
@@ -264,6 +264,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete)
|
||||
s3VectorsApi.HandleFunc("/indexes", h.adminServer.GetS3VectorsBucketAPI).Methods(http.MethodGet)
|
||||
s3VectorsApi.Handle("/indexes", wrapWrite(h.adminServer.DeleteS3VectorsIndex)).Methods(http.MethodDelete)
|
||||
s3VectorsApi.Handle("/metadata-keys", wrapWrite(h.adminServer.UpdateS3VectorsMetadataKeyAPI)).Methods(http.MethodPost, http.MethodDelete)
|
||||
|
||||
filesApi := api.PathPrefix("/files").Subrouter()
|
||||
filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete)
|
||||
|
||||
@@ -99,6 +99,7 @@ var jobTypeLaneMap = map[string]SchedulerLane{
|
||||
"lance_optimize_indices": LaneLance,
|
||||
"lance_cleanup_versions": LaneLance,
|
||||
"lance_vector_index": LaneLance,
|
||||
"lance_metadata_index": LaneLance,
|
||||
}
|
||||
|
||||
// JobTypeLane returns the scheduler lane for the given job type.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
@@ -103,6 +104,9 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
for _, panel := range data.Panels {
|
||||
@S3VectorsIndexPanel(panel)
|
||||
}
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
@@ -138,6 +142,52 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="indexS3VectorsMetadataKeyModal" tabindex="-1" aria-labelledby="indexS3VectorsMetadataKeyModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="indexS3VectorsMetadataKeyModalLabel">
|
||||
<i class="bi bi-tags me-2"></i>Index a metadata key of <span id="indexS3VectorsMetadataKeyIndex"></span>
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<label for="indexS3VectorsMetadataKeyName" class="form-label">Metadata key</label>
|
||||
<input type="text" class="form-control" id="indexS3VectorsMetadataKeyName" autocomplete="off"/>
|
||||
<div class="form-text">Filters comparing this key with strings ($eq, $in) read its index instead of every row's metadata. Results do not change.</div>
|
||||
<div class="text-danger small mt-2" id="indexS3VectorsMetadataKeyError"></div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-primary" id="indexS3VectorsMetadataKeyBtn">
|
||||
<i class="bi bi-plus-lg me-1"></i>Index
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="removeS3VectorsMetadataKeyModal" tabindex="-1" aria-labelledby="removeS3VectorsMetadataKeyModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="removeS3VectorsMetadataKeyModalLabel">
|
||||
<i class="bi bi-exclamation-triangle me-2 text-warning"></i><span id="removeS3VectorsMetadataKeyTitle">Remove Metadata Index</span>
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p><span id="removeS3VectorsMetadataKeyAction">Remove</span> the index of metadata key <strong id="removeS3VectorsMetadataKeyName"></strong> on index <strong id="removeS3VectorsMetadataKeyIndex"></strong>? Filters on the key read every row's metadata again.</p>
|
||||
<div class="text-danger small mt-2" id="removeS3VectorsMetadataKeyError"></div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<button type="button" class="btn btn-danger" id="removeS3VectorsMetadataKeyBtn">
|
||||
<i class="bi bi-trash me-1"></i><span id="removeS3VectorsMetadataKeyBtnLabel">Remove</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="s3vectorsBucketDetails" data-bucket-name={ data.Bucket.Name }></div>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
@@ -151,6 +201,83 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
modal.show();
|
||||
});
|
||||
});
|
||||
async function updateMetadataKey(index, key, remove) {
|
||||
const url = remove
|
||||
? '/api/s3vectors/metadata-keys?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index) + '&key=' + encodeURIComponent(key)
|
||||
: '/api/s3vectors/metadata-keys';
|
||||
const options = { method: remove ? 'DELETE' : 'POST', headers: s3tWriteHeaders() };
|
||||
if (!remove) {
|
||||
options.headers = Object.assign({ 'Content-Type': 'application/json' }, options.headers);
|
||||
options.body = JSON.stringify({ bucket: bucket, index: index, key: key });
|
||||
}
|
||||
const response = await fetch(s3tBasePath(url), options);
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
throw new Error(data.error || 'request failed');
|
||||
}
|
||||
return data;
|
||||
}
|
||||
const keyModalEl = document.getElementById('indexS3VectorsMetadataKeyModal');
|
||||
const keyModal = new bootstrap.Modal(keyModalEl);
|
||||
const keyInput = document.getElementById('indexS3VectorsMetadataKeyName');
|
||||
const keyError = document.getElementById('indexS3VectorsMetadataKeyError');
|
||||
document.querySelectorAll('.s3vectors-index-key-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
keyModalEl.dataset.indexName = this.dataset.indexName || '';
|
||||
keyModalEl.dataset.nonFilterable = this.dataset.nonFilterable || '[]';
|
||||
document.getElementById('indexS3VectorsMetadataKeyIndex').textContent = this.dataset.indexName || '';
|
||||
keyInput.value = this.dataset.key || '';
|
||||
keyError.textContent = '';
|
||||
keyModal.show();
|
||||
});
|
||||
});
|
||||
document.getElementById('indexS3VectorsMetadataKeyBtn').addEventListener('click', async function () {
|
||||
const index = keyModalEl.dataset.indexName;
|
||||
// Metadata key names are exact strings: only an empty name is refused.
|
||||
const key = keyInput.value;
|
||||
if (!key.trim()) {
|
||||
keyError.textContent = 'Enter a metadata key.';
|
||||
return;
|
||||
}
|
||||
if (JSON.parse(keyModalEl.dataset.nonFilterable || '[]').includes(key)) {
|
||||
keyError.textContent = key + ' is a non-filterable key of this index and cannot be indexed.';
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await updateMetadataKey(index, key, false);
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
keyError.textContent = error.message;
|
||||
}
|
||||
});
|
||||
const removeModalEl = document.getElementById('removeS3VectorsMetadataKeyModal');
|
||||
const removeModal = new bootstrap.Modal(removeModalEl);
|
||||
const removeError = document.getElementById('removeS3VectorsMetadataKeyError');
|
||||
document.querySelectorAll('.s3vectors-remove-key-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
const action = this.dataset.action || 'Remove';
|
||||
removeModalEl.dataset.indexName = this.dataset.indexName || '';
|
||||
removeModalEl.dataset.key = this.dataset.key || '';
|
||||
document.getElementById('removeS3VectorsMetadataKeyTitle').textContent = action + ' Metadata Index';
|
||||
document.getElementById('removeS3VectorsMetadataKeyAction').textContent = action;
|
||||
document.getElementById('removeS3VectorsMetadataKeyBtnLabel').textContent = action;
|
||||
document.getElementById('removeS3VectorsMetadataKeyName').textContent = this.dataset.key || '';
|
||||
document.getElementById('removeS3VectorsMetadataKeyIndex').textContent = this.dataset.indexName || '';
|
||||
removeError.textContent = '';
|
||||
removeModal.show();
|
||||
});
|
||||
});
|
||||
document.getElementById('removeS3VectorsMetadataKeyBtn').addEventListener('click', async function () {
|
||||
const index = removeModalEl.dataset.indexName;
|
||||
const key = removeModalEl.dataset.key;
|
||||
if (!index || !key) return;
|
||||
try {
|
||||
await updateMetadataKey(index, key, true);
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
removeError.textContent = error.message;
|
||||
}
|
||||
});
|
||||
document.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {
|
||||
const index = modalEl.dataset.indexName;
|
||||
if (!bucket || !index) return;
|
||||
@@ -170,6 +297,144 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
</script>
|
||||
}
|
||||
|
||||
templ S3VectorsIndexPanel(panel dash.S3VectorsIndexPanel) {
|
||||
<div class="card shadow mb-4 s3vectors-index-panel" data-index-name={ panel.Index.IndexName }>
|
||||
<div class="card-header py-3">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="bi bi-diagram-3 me-2"></i>Index: { panel.Index.IndexName }
|
||||
</h6>
|
||||
<div class="small text-muted mt-1">
|
||||
{ fmt.Sprintf("Dimension %d", panel.Index.Dimension) } · { panel.Index.DistanceMetric } · { groupDigits(float64(panel.Vectors)) } vectors · Vector index:
|
||||
if panel.VectorIndex != "" {
|
||||
{ panel.VectorIndex }
|
||||
} else {
|
||||
none (exact scan)
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
if panel.Error != "" {
|
||||
<div class="alert alert-warning py-2 small">{ panel.Error }</div>
|
||||
}
|
||||
<div class="d-flex justify-content-between align-items-center mb-2">
|
||||
<strong>Metadata indexes</strong>
|
||||
<button type="button" class="btn btn-outline-primary btn-sm s3vectors-index-key-btn" data-index-name={ panel.Index.IndexName } data-non-filterable={ nonFilterableJSON(panel.Index) }>
|
||||
<i class="bi bi-plus-lg me-1"></i>Index a key
|
||||
</button>
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm table-hover mb-3">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Key</th>
|
||||
<th>State</th>
|
||||
<th>Coverage</th>
|
||||
<th>Filtered queries/h</th>
|
||||
<th>Size</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, key := range panel.Keys {
|
||||
<tr>
|
||||
<td><code>{ key.Key }</code></td>
|
||||
<td>{ keyState(key.State) }</td>
|
||||
<td>
|
||||
if key.State == "requested" {
|
||||
—
|
||||
} else {
|
||||
{ fmt.Sprintf("%.0f%%", key.Coverage) }
|
||||
}
|
||||
</td>
|
||||
<td>{ groupDigits(key.QueriesPerHour) }</td>
|
||||
<td>
|
||||
if key.IndexBytes > 0 {
|
||||
{ fmt.Sprintf("%.1f MiB", float64(key.IndexBytes)/(1<<20)) }
|
||||
} else {
|
||||
—
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
if key.State == "ready" {
|
||||
<button type="button" class="btn btn-link btn-sm p-0 text-danger s3vectors-remove-key-btn" data-index-name={ panel.Index.IndexName } data-key={ key.Key } data-action="Remove">Remove</button>
|
||||
} else if key.State == "requested" || key.State == "building" {
|
||||
<button type="button" class="btn btn-link btn-sm p-0 s3vectors-remove-key-btn" data-index-name={ panel.Index.IndexName } data-key={ key.Key } data-action="Cancel">Cancel</button>
|
||||
}
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(panel.Keys) == 0 {
|
||||
<tr>
|
||||
<td colspan="6" class="text-center text-muted">No indexed metadata keys.</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<strong>Frequently filtered, not indexed</strong>
|
||||
if panel.StatsError != "" {
|
||||
<div class="text-muted small">{ panel.StatsError }</div>
|
||||
}
|
||||
if panel.StatsTruncated {
|
||||
<div class="text-muted small">Partial: an engine counts a limited number of keys per index, so some filtered keys are missing.</div>
|
||||
}
|
||||
if len(panel.Frequent) == 0 {
|
||||
<div class="text-muted small mt-1">No filtered queries in the last hour on keys without an index.</div>
|
||||
}
|
||||
<ul class="list-unstyled mt-1 mb-0">
|
||||
for _, key := range panel.Frequent {
|
||||
<li class="d-flex align-items-center py-1">
|
||||
<code class="me-3">{ key.Key }</code>
|
||||
<span class="me-3">{ groupDigits(key.QueriesPerHour) } queries/h</span>
|
||||
<span class="me-3 text-muted">{ fmt.Sprintf("avg %.0f ms", key.AvgMs) }</span>
|
||||
if key.Numeric {
|
||||
<span class="text-muted small">(numeric: phase 2)</span>
|
||||
} else {
|
||||
<button type="button" class="btn btn-outline-primary btn-sm ms-auto s3vectors-index-key-btn" data-index-name={ panel.Index.IndexName } data-key={ key.Key } data-non-filterable={ nonFilterableJSON(panel.Index) }>Index</button>
|
||||
}
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
func keyState(state string) string {
|
||||
switch state {
|
||||
case "requested":
|
||||
return "Requested"
|
||||
case "building":
|
||||
return "Building"
|
||||
case "ready":
|
||||
return "Ready"
|
||||
case "off":
|
||||
return "Removing"
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// groupDigits renders n rounded, with thousands separators.
|
||||
func groupDigits(n float64) string {
|
||||
digits := fmt.Sprintf("%.0f", n)
|
||||
var out []byte
|
||||
for i := range digits {
|
||||
if i > 0 && (len(digits)-i)%3 == 0 {
|
||||
out = append(out, ',')
|
||||
}
|
||||
out = append(out, digits[i])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func nonFilterableJSON(index *s3vectors.Index) string {
|
||||
keys := []string{}
|
||||
if index.MetadataConfiguration != nil && index.MetadataConfiguration.NonFilterableMetadataKeys != nil {
|
||||
keys = index.MetadataConfiguration.NonFilterableMetadataKeys
|
||||
}
|
||||
b, _ := json.Marshal(keys)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
const maxIndentedJSONDepth = 32
|
||||
|
||||
func indentJSON(s string) string {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
@@ -42,7 +43,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 16, Col: 63}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 17, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -55,7 +56,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 20, Col: 89}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 21, Col: 89}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -68,7 +69,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 21, Col: 78}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 22, Col: 78}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -93,7 +94,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 49}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 33, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -106,7 +107,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 34, Col: 79}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 35, Col: 79}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -124,7 +125,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 38, Col: 61}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 39, Col: 61}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -147,7 +148,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 84}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 50, Col: 84}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -165,7 +166,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 51, Col: 138}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 52, Col: 138}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -188,7 +189,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 73, Col: 26}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 26}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -201,7 +202,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var11 string
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 55}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -214,7 +215,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var12 string
|
||||
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 28}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 77, Col: 28}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -227,7 +228,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var13 string
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 77, Col: 48}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 78, Col: 48}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -240,7 +241,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var14 string
|
||||
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 78, Col: 34}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 34}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -258,7 +259,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var15 string
|
||||
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", "))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 81, Col: 91}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 82, Col: 91}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -281,7 +282,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 59}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 87, Col: 59}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -294,7 +295,7 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
var templ_7745c5c3_Var17 string
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templruntime.NewAttributeExpression(index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 88, Col: 129}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 89, Col: 129}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -311,48 +312,58 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</tbody></table></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</tbody></table></div></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, panel := range data.Panels {
|
||||
templ_7745c5c3_Err = S3VectorsIndexPanel(panel).Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.Policy != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var18 string
|
||||
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 114, Col: 102}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 118, Col: 102}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "</pre>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</pre>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div class=\"modal fade\" id=\"indexS3VectorsMetadataKeyModal\" tabindex=\"-1\" aria-labelledby=\"indexS3VectorsMetadataKeyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"indexS3VectorsMetadataKeyModalLabel\"><i class=\"bi bi-tags me-2\"></i>Index a metadata key of <span id=\"indexS3VectorsMetadataKeyIndex\"></span></h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><label for=\"indexS3VectorsMetadataKeyName\" class=\"form-label\">Metadata key</label> <input type=\"text\" class=\"form-control\" id=\"indexS3VectorsMetadataKeyName\" autocomplete=\"off\"><div class=\"form-text\">Filters comparing this key with strings ($eq, $in) read its index instead of every row's metadata. Results do not change.</div><div class=\"text-danger small mt-2\" id=\"indexS3VectorsMetadataKeyError\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" id=\"indexS3VectorsMetadataKeyBtn\"><i class=\"bi bi-plus-lg me-1\"></i>Index</button></div></div></div></div><div class=\"modal fade\" id=\"removeS3VectorsMetadataKeyModal\" tabindex=\"-1\" aria-labelledby=\"removeS3VectorsMetadataKeyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"removeS3VectorsMetadataKeyModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i><span id=\"removeS3VectorsMetadataKeyTitle\">Remove Metadata Index</span></h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p><span id=\"removeS3VectorsMetadataKeyAction\">Remove</span> the index of metadata key <strong id=\"removeS3VectorsMetadataKeyName\"></strong> on index <strong id=\"removeS3VectorsMetadataKeyIndex\"></strong>? Filters on the key read every row's metadata again.</p><div class=\"text-danger small mt-2\" id=\"removeS3VectorsMetadataKeyError\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-danger\" id=\"removeS3VectorsMetadataKeyBtn\"><i class=\"bi bi-trash me-1\"></i><span id=\"removeS3VectorsMetadataKeyBtnLabel\">Remove</span></button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var19 string
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.Bucket.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 141, Col: 69}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 191, Col: 69}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var19)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = modalEl.dataset.indexName;\n\t\t\t\tif (!bucket || !index) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete index');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete index: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tasync function updateMetadataKey(index, key, remove) {\n\t\t\t\tconst url = remove\n\t\t\t\t\t? '/api/s3vectors/metadata-keys?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index) + '&key=' + encodeURIComponent(key)\n\t\t\t\t\t: '/api/s3vectors/metadata-keys';\n\t\t\t\tconst options = { method: remove ? 'DELETE' : 'POST', headers: s3tWriteHeaders() };\n\t\t\t\tif (!remove) {\n\t\t\t\t\toptions.headers = Object.assign({ 'Content-Type': 'application/json' }, options.headers);\n\t\t\t\t\toptions.body = JSON.stringify({ bucket: bucket, index: index, key: key });\n\t\t\t\t}\n\t\t\t\tconst response = await fetch(s3tBasePath(url), options);\n\t\t\t\tconst data = await response.json();\n\t\t\t\tif (!response.ok) {\n\t\t\t\t\tthrow new Error(data.error || 'request failed');\n\t\t\t\t}\n\t\t\t\treturn data;\n\t\t\t}\n\t\t\tconst keyModalEl = document.getElementById('indexS3VectorsMetadataKeyModal');\n\t\t\tconst keyModal = new bootstrap.Modal(keyModalEl);\n\t\t\tconst keyInput = document.getElementById('indexS3VectorsMetadataKeyName');\n\t\t\tconst keyError = document.getElementById('indexS3VectorsMetadataKeyError');\n\t\t\tdocument.querySelectorAll('.s3vectors-index-key-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tkeyModalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tkeyModalEl.dataset.nonFilterable = this.dataset.nonFilterable || '[]';\n\t\t\t\t\tdocument.getElementById('indexS3VectorsMetadataKeyIndex').textContent = this.dataset.indexName || '';\n\t\t\t\t\tkeyInput.value = this.dataset.key || '';\n\t\t\t\t\tkeyError.textContent = '';\n\t\t\t\t\tkeyModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('indexS3VectorsMetadataKeyBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = keyModalEl.dataset.indexName;\n\t\t\t\t// Metadata key names are exact strings: only an empty name is refused.\n\t\t\t\tconst key = keyInput.value;\n\t\t\t\tif (!key.trim()) {\n\t\t\t\t\tkeyError.textContent = 'Enter a metadata key.';\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\tif (JSON.parse(keyModalEl.dataset.nonFilterable || '[]').includes(key)) {\n\t\t\t\t\tkeyError.textContent = key + ' is a non-filterable key of this index and cannot be indexed.';\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\ttry {\n\t\t\t\t\tawait updateMetadataKey(index, key, false);\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tkeyError.textContent = error.message;\n\t\t\t\t}\n\t\t\t});\n\t\t\tconst removeModalEl = document.getElementById('removeS3VectorsMetadataKeyModal');\n\t\t\tconst removeModal = new bootstrap.Modal(removeModalEl);\n\t\t\tconst removeError = document.getElementById('removeS3VectorsMetadataKeyError');\n\t\t\tdocument.querySelectorAll('.s3vectors-remove-key-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tconst action = this.dataset.action || 'Remove';\n\t\t\t\t\tremoveModalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tremoveModalEl.dataset.key = this.dataset.key || '';\n\t\t\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyTitle').textContent = action + ' Metadata Index';\n\t\t\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyAction').textContent = action;\n\t\t\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyBtnLabel').textContent = action;\n\t\t\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyName').textContent = this.dataset.key || '';\n\t\t\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyIndex').textContent = this.dataset.indexName || '';\n\t\t\t\t\tremoveError.textContent = '';\n\t\t\t\t\tremoveModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('removeS3VectorsMetadataKeyBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = removeModalEl.dataset.indexName;\n\t\t\t\tconst key = removeModalEl.dataset.key;\n\t\t\t\tif (!index || !key) return;\n\t\t\t\ttry {\n\t\t\t\t\tawait updateMetadataKey(index, key, true);\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tremoveError.textContent = error.message;\n\t\t\t\t}\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async functionLine truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -360,6 +371,501 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
})
|
||||
}
|
||||
|
||||
func S3VectorsIndexPanel(panel dash.S3VectorsIndexPanel) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "<div class=\"card shadow mb-4 s3vectors-index-panel\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var21 string
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templruntime.NewAttributeExpression(panel.Index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 301, Col: 92}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var21)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-diagram-3 me-2\"></i>Index: ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var22 string
|
||||
templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(panel.Index.IndexName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 304, Col: 70}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "</h6><div class=\"small text-muted mt-1\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var23 string
|
||||
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("Dimension %d", panel.Index.Dimension))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 307, Col: 56}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, " · ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var24 string
|
||||
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(panel.Index.DistanceMetric)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 307, Col: 90}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, " · ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var25 string
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(groupDigits(float64(panel.Vectors)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 307, Col: 133}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, " vectors · Vector index: ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if panel.VectorIndex != "" {
|
||||
var templ_7745c5c3_Var26 string
|
||||
templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(panel.VectorIndex)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 309, Col: 24}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "none (exact scan)")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "</div></div><div class=\"card-body\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if panel.Error != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<div class=\"alert alert-warning py-2 small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var27 string
|
||||
templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(panel.Error)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 317, Col: 61}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<div class=\"d-flex justify-content-between align-items-center mb-2\"><strong>Metadata indexes</strong> <button type=\"button\" class=\"btn btn-outline-primary btn-sm s3vectors-index-key-btn\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var28 string
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templruntime.NewAttributeExpression(panel.Index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 321, Col: 128}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var28)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" data-non-filterable=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var29 string
|
||||
templ_7745c5c3_Var29, templ_7745c5c3_Err = templruntime.NewAttributeExpression(nonFilterableJSON(panel.Index)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 321, Col: 183}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\"><i class=\"bi bi-plus-lg me-1\"></i>Index a key</button></div><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-3\"><thead><tr><th>Key</th><th>State</th><th>Coverage</th><th>Filtered queries/h</th><th>Size</th><th></th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, key := range panel.Keys {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "<tr><td><code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var30 string
|
||||
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(key.Key)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 340, Col: 27}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "</code></td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var31 string
|
||||
templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(keyState(key.State))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 341, Col: 33}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if key.State == "requested" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "—")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
var templ_7745c5c3_Var32 string
|
||||
templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f%%", key.Coverage))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 346, Col: 47}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var33 string
|
||||
templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(groupDigits(key.QueriesPerHour))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 349, Col: 45}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if key.IndexBytes > 0 {
|
||||
var templ_7745c5c3_Var34 string
|
||||
templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f MiB", float64(key.IndexBytes)/(1<<20)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 352, Col: 68}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "—")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if key.State == "ready" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "<button type=\"button\" class=\"btn btn-link btn-sm p-0 text-danger s3vectors-remove-key-btn\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var35 string
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(panel.Index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 359, Col: 140}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var35)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\" data-key=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var36 string
|
||||
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(key.Key).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 359, Col: 161}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var36)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "\" data-action=\"Remove\">Remove</button>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else if key.State == "requested" || key.State == "building" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "<button type=\"button\" class=\"btn btn-link btn-sm p-0 s3vectors-remove-key-btn\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var37 string
|
||||
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(panel.Index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 361, Col: 128}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "\" data-key=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var38 string
|
||||
templ_7745c5c3_Var38, templ_7745c5c3_Err = templruntime.NewAttributeExpression(key.Key).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 361, Col: 149}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var38)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "\" data-action=\"Cancel\">Cancel</button>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(panel.Keys) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<tr><td colspan=\"6\" class=\"text-center text-muted\">No indexed metadata keys.</td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "</tbody></table></div><strong>Frequently filtered, not indexed</strong> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if panel.StatsError != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<div class=\"text-muted small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var39 string
|
||||
templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(panel.StatsError)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 376, Col: 52}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if panel.StatsTruncated {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "<div class=\"text-muted small\">Partial: an engine counts a limited number of keys per index, so some filtered keys are missing.</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(panel.Frequent) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "<div class=\"text-muted small mt-1\">No filtered queries in the last hour on keys without an index.</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<ul class=\"list-unstyled mt-1 mb-0\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, key := range panel.Frequent {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "<li class=\"d-flex align-items-center py-1\"><code class=\"me-3\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var40 string
|
||||
templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(key.Key)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 387, Col: 34}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var40))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "</code> <span class=\"me-3\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var41 string
|
||||
templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(groupDigits(key.QueriesPerHour))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 388, Col: 58}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, " queries/h</span> <span class=\"me-3 text-muted\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var42 string
|
||||
templ_7745c5c3_Var42, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("avg %.0f ms", key.AvgMs))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 389, Col: 75}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var42))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if key.Numeric {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "<span class=\"text-muted small\">(numeric: phase 2)</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm ms-auto s3vectors-index-key-btn\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var43 string
|
||||
templ_7745c5c3_Var43, templ_7745c5c3_Err = templruntime.NewAttributeExpression(panel.Index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 393, Col: 139}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var43)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "\" data-key=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var44 string
|
||||
templ_7745c5c3_Var44, templ_7745c5c3_Err = templruntime.NewAttributeExpression(key.Key).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 393, Col: 160}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var44)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "\" data-non-filterable=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var45 string
|
||||
templ_7745c5c3_Var45, templ_7745c5c3_Err = templruntime.NewAttributeExpression(nonFilterableJSON(panel.Index)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 393, Col: 215}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var45)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "\">Index</button>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "</li>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, "</ul></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func keyState(state string) string {
|
||||
switch state {
|
||||
case "requested":
|
||||
return "Requested"
|
||||
case "building":
|
||||
return "Building"
|
||||
case "ready":
|
||||
return "Ready"
|
||||
case "off":
|
||||
return "Removing"
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// groupDigits renders n rounded, with thousands separators.
|
||||
func groupDigits(n float64) string {
|
||||
digits := fmt.Sprintf("%.0f", n)
|
||||
var out []byte
|
||||
for i := range digits {
|
||||
if i > 0 && (len(digits)-i)%3 == 0 {
|
||||
out = append(out, ',')
|
||||
}
|
||||
out = append(out, digits[i])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func nonFilterableJSON(index *s3vectors.Index) string {
|
||||
keys := []string{}
|
||||
if index.MetadataConfiguration != nil && index.MetadataConfiguration.NonFilterableMetadataKeys != nil {
|
||||
keys = index.MetadataConfiguration.NonFilterableMetadataKeys
|
||||
}
|
||||
b, _ := json.Marshal(keys)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
const maxIndentedJSONDepth = 32
|
||||
|
||||
func indentJSON(s string) string {
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
func TestGroupDigits(t *testing.T) {
|
||||
for n, want := range map[float64]string{0: "0", 999: "999", 1000: "1,000", 12400: "12,400", 2140312: "2,140,312", 8899.6: "8,900"} {
|
||||
if got := groupDigits(n); got != want {
|
||||
t.Errorf("groupDigits(%v) = %q, want %q", n, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonFilterableJSONIsAlwaysAList(t *testing.T) {
|
||||
for _, index := range []*s3vectors.Index{
|
||||
{},
|
||||
{MetadataConfiguration: &s3vectors.MetadataConfiguration{}},
|
||||
{MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{}}},
|
||||
} {
|
||||
if got := nonFilterableJSON(index); got != "[]" {
|
||||
t.Errorf("%+v: %s", index.MetadataConfiguration, got)
|
||||
}
|
||||
}
|
||||
index := &s3vectors.Index{MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}}
|
||||
if got := nonFilterableJSON(index); got != `["body"]` {
|
||||
t.Error(got)
|
||||
}
|
||||
}
|
||||
|
||||
// Removing or cancelling a key's index asks in a Bootstrap modal, as
|
||||
// deleting an index does, not in the browser's confirm().
|
||||
func TestMetadataKeyRemovalAsksInAModal(t *testing.T) {
|
||||
data := dash.S3VectorsBucketData{
|
||||
Bucket: dash.S3VectorsBucketSummary{Name: "b"},
|
||||
Panels: []dash.S3VectorsIndexPanel{{
|
||||
Index: &s3vectors.Index{IndexName: "i"},
|
||||
Keys: []dash.S3VectorsMetadataKeyRow{
|
||||
{Key: "author", State: "ready"},
|
||||
{Key: "category", State: "building"},
|
||||
},
|
||||
}},
|
||||
}
|
||||
var page strings.Builder
|
||||
if err := S3VectorsBucketDetails(data).Render(context.Background(), &page); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
html := page.String()
|
||||
if strings.Contains(html, "' the index of metadata key ' +") {
|
||||
t.Error("removing a key still asks with confirm()")
|
||||
}
|
||||
for _, want := range []string{
|
||||
`id="removeS3VectorsMetadataKeyModal"`,
|
||||
`id="removeS3VectorsMetadataKeyBtn"`,
|
||||
`new bootstrap.Modal(removeModalEl)`,
|
||||
`data-key="author" data-action="Remove"`,
|
||||
`data-key="category" data-action="Cancel"`,
|
||||
} {
|
||||
if !strings.Contains(html, want) {
|
||||
t.Errorf("missing %s", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,16 +74,8 @@ func (c *commandS3Vectors) Do(args []string, env *shell.CommandEnv, writer io.Wr
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
backend := s3vectors.NewVectorDiscovery(env.MasterClient.ListCachedMasters, env.GrpcDialOption(), env.MasterClient.FilerGroup)
|
||||
defer backend.Close()
|
||||
server := s3vectors.NewServer(env, nil, backend)
|
||||
server.SetRegion(s3tables.DefaultRegion)
|
||||
server.SetAccountID(s3tables.DefaultAccountID)
|
||||
catalog := lance.NewServer(env, nil)
|
||||
server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
|
||||
r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
|
||||
return catalog.DescribeTableInternal(r, id)
|
||||
})
|
||||
server, closeServer := shellVectorsServer(env)
|
||||
defer closeServer()
|
||||
var response json.RawMessage
|
||||
if err := server.Execute(ctx, *operation, json.RawMessage(body), &response, s3_constants.AccountAdminId); err != nil {
|
||||
return err
|
||||
@@ -99,3 +91,18 @@ func (c *commandS3Vectors) Do(args []string, env *shell.CommandEnv, writer io.Wr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shellVectorsServer is the S3 Vectors server the shell runs in process, on
|
||||
// trusted cluster credentials, and how to close its engine connection.
|
||||
func shellVectorsServer(env *shell.CommandEnv) (*s3vectors.Server, func()) {
|
||||
backend := s3vectors.NewVectorDiscovery(env.MasterClient.ListCachedMasters, env.GrpcDialOption(), env.MasterClient.FilerGroup)
|
||||
server := s3vectors.NewServer(env, nil, backend)
|
||||
server.SetRegion(s3tables.DefaultRegion)
|
||||
server.SetAccountID(s3tables.DefaultAccountID)
|
||||
catalog := lance.NewServer(env, nil)
|
||||
server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
|
||||
r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
|
||||
return catalog.DescribeTableInternal(r, id)
|
||||
})
|
||||
return server, func() { backend.Close() }
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/shell"
|
||||
)
|
||||
|
||||
func init() { shell.Commands = append(shell.Commands, &commandS3VectorsIndexMetadata{}) }
|
||||
|
||||
type commandS3VectorsIndexMetadata struct{}
|
||||
|
||||
func (c *commandS3VectorsIndexMetadata) Name() string { return "s3vectors.index.metadata" }
|
||||
func (c *commandS3VectorsIndexMetadata) HasTag(shell.CommandTag) bool { return false }
|
||||
func (c *commandS3VectorsIndexMetadata) Help() string {
|
||||
return `list, add or remove the indexed metadata keys of a vector index
|
||||
|
||||
s3vectors.index.metadata -bucket embeddings -index articles
|
||||
s3vectors.index.metadata -bucket embeddings -index articles -add author
|
||||
s3vectors.index.metadata -bucket embeddings -index articles -remove author
|
||||
|
||||
An indexed key keeps a scalar index of the string values a filter's $eq and
|
||||
$in compare, so those filters stop reading every row's metadata. Query
|
||||
results do not change. A key is first "requested"; the lance_metadata_index
|
||||
worker job builds it ("building") and then "ready" queries use it. Removing
|
||||
sets it "off" at once and the job drops it. Non-filterable keys cannot be
|
||||
indexed, an index has at most 8 indexed keys, and adding is refused while a
|
||||
registered vector engine does not fill indexed key columns on its writes.
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandS3VectorsIndexMetadata) Do(args []string, env *shell.CommandEnv, writer io.Writer) error {
|
||||
flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
bucket := flags.String("bucket", "", "vector bucket name")
|
||||
index := flags.String("index", "", "index name")
|
||||
add := flags.String("add", "", "metadata key to index")
|
||||
remove := flags.String("remove", "", "metadata key to stop indexing")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *bucket == "" || *index == "" {
|
||||
return fmt.Errorf("-bucket and -index are required")
|
||||
}
|
||||
if *add != "" && *remove != "" {
|
||||
return fmt.Errorf("use either -add or -remove")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
server, closeServer := shellVectorsServer(env)
|
||||
defer closeServer()
|
||||
var keys *vector_pb.MetadataKeysResponse
|
||||
var err error
|
||||
switch {
|
||||
case *add != "":
|
||||
keys, err = server.UpdateMetadataKey(ctx, *bucket, *index, *add, false)
|
||||
case *remove != "":
|
||||
keys, err = server.UpdateMetadataKey(ctx, *bucket, *index, *remove, true)
|
||||
default:
|
||||
keys, err = server.MetadataKeys(ctx, *bucket, *index)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printMetadataKeys(writer, keys)
|
||||
}
|
||||
|
||||
func printMetadataKeys(writer io.Writer, keys *vector_pb.MetadataKeysResponse) error {
|
||||
if len(keys.Keys) == 0 {
|
||||
_, err := fmt.Fprintln(writer, "no indexed metadata keys")
|
||||
return err
|
||||
}
|
||||
w := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "KEY\tSTATE\tCOVERAGE\tSEGMENTS\tSIZE")
|
||||
for _, k := range keys.Keys {
|
||||
coverage := "-"
|
||||
if k.Rows > 0 && k.State != "requested" {
|
||||
coverage = fmt.Sprintf("%d%%", k.IndexedRows*100/k.Rows)
|
||||
}
|
||||
size := "-"
|
||||
if k.IndexBytes > 0 {
|
||||
size = fmt.Sprintf("%.1f MiB", float64(k.IndexBytes)/(1<<20))
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\n", k.Key, k.State, coverage, k.Segments, size)
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
||||
)
|
||||
|
||||
func TestPrintMetadataKeys(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := printMetadataKeys(&out, &vector_pb.MetadataKeysResponse{Keys: []*vector_pb.MetadataKey{
|
||||
{Key: "lang", State: "requested", Rows: 10},
|
||||
{Key: "author", State: "ready", Column: "_mk_0", IndexedRows: 75, Rows: 100, Segments: 2, IndexBytes: 3 << 20},
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "requested") || !strings.Contains(lines[2], "75%") || !strings.Contains(lines[2], "3.0 MiB") {
|
||||
t.Fatalf("%q", out.String())
|
||||
}
|
||||
out.Reset()
|
||||
_ = printMetadataKeys(&out, &vector_pb.MetadataKeysResponse{})
|
||||
if !strings.Contains(out.String(), "no indexed metadata keys") {
|
||||
t.Fatal(out.String())
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,11 @@ service VectorService {
|
||||
// How this engine's queries of an index filtered on each metadata key
|
||||
// over the last hour.
|
||||
rpc FilterStats(FilterStatsRequest) returns (FilterStatsResponse);
|
||||
// The index's indexed metadata keys, as its Lance table records them.
|
||||
rpc MetadataKeys(MetadataKeysRequest) returns (MetadataKeysResponse);
|
||||
// Ask for a metadata key to be indexed, or no longer: the table records
|
||||
// the request and the maintenance worker carries it out.
|
||||
rpc UpdateMetadataKey(UpdateMetadataKeyRequest) returns (MetadataKeysResponse);
|
||||
}
|
||||
|
||||
enum DistanceMetric {
|
||||
@@ -167,3 +172,34 @@ message FilterStatsResponse {
|
||||
// number of them.
|
||||
bool truncated = 3;
|
||||
}
|
||||
|
||||
message MetadataKeysRequest {
|
||||
Table table = 1;
|
||||
}
|
||||
|
||||
message MetadataKey {
|
||||
string key = 1;
|
||||
// requested, building, ready or off.
|
||||
string state = 2;
|
||||
// The hidden column holding the key; empty while requested.
|
||||
string column = 3;
|
||||
// Rows the key's scalar index covers, of the table's rows.
|
||||
uint64 indexed_rows = 4;
|
||||
uint64 rows = 5;
|
||||
uint64 index_bytes = 6;
|
||||
uint32 segments = 7;
|
||||
}
|
||||
|
||||
message MetadataKeysResponse {
|
||||
repeated MetadataKey keys = 1;
|
||||
uint64 version = 2;
|
||||
// The table's rows, and its vector index type (empty without one).
|
||||
uint64 rows = 3;
|
||||
string vector_index = 4;
|
||||
}
|
||||
|
||||
message UpdateMetadataKeyRequest {
|
||||
Table table = 1;
|
||||
string key = 2;
|
||||
bool remove = 3;
|
||||
}
|
||||
+351
-50
@@ -1271,6 +1271,274 @@ func (x *FilterStatsResponse) GetTruncated() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
type MetadataKeysRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Table *Table `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *MetadataKeysRequest) Reset() {
|
||||
*x = MetadataKeysRequest{}
|
||||
mi := &file_vector_proto_msgTypes[20]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *MetadataKeysRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*MetadataKeysRequest) ProtoMessage() {}
|
||||
|
||||
func (x *MetadataKeysRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_vector_proto_msgTypes[20]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use MetadataKeysRequest.ProtoReflect.Descriptor instead.
|
||||
func (*MetadataKeysRequest) Descriptor() ([]byte, []int) {
|
||||
return file_vector_proto_rawDescGZIP(), []int{20}
|
||||
}
|
||||
|
||||
func (x *MetadataKeysRequest) GetTable() *Table {
|
||||
if x != nil {
|
||||
return x.Table
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type MetadataKey struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
|
||||
// requested, building, ready or off.
|
||||
State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"`
|
||||
// The hidden column holding the key; empty while requested.
|
||||
Column string `protobuf:"bytes,3,opt,name=column,proto3" json:"column,omitempty"`
|
||||
// Rows the key's scalar index covers, of the table's rows.
|
||||
IndexedRows uint64 `protobuf:"varint,4,opt,name=indexed_rows,json=indexedRows,proto3" json:"indexed_rows,omitempty"`
|
||||
Rows uint64 `protobuf:"varint,5,opt,name=rows,proto3" json:"rows,omitempty"`
|
||||
IndexBytes uint64 `protobuf:"varint,6,opt,name=index_bytes,json=indexBytes,proto3" json:"index_bytes,omitempty"`
|
||||
Segments uint32 `protobuf:"varint,7,opt,name=segments,proto3" json:"segments,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *MetadataKey) Reset() {
|
||||
*x = MetadataKey{}
|
||||
mi := &file_vector_proto_msgTypes[21]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *MetadataKey) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*MetadataKey) ProtoMessage() {}
|
||||
|
||||
func (x *MetadataKey) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_vector_proto_msgTypes[21]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use MetadataKey.ProtoReflect.Descriptor instead.
|
||||
func (*MetadataKey) Descriptor() ([]byte, []int) {
|
||||
return file_vector_proto_rawDescGZIP(), []int{21}
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetKey() string {
|
||||
if x != nil {
|
||||
return x.Key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetState() string {
|
||||
if x != nil {
|
||||
return x.State
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetColumn() string {
|
||||
if x != nil {
|
||||
return x.Column
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetIndexedRows() uint64 {
|
||||
if x != nil {
|
||||
return x.IndexedRows
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetRows() uint64 {
|
||||
if x != nil {
|
||||
return x.Rows
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetIndexBytes() uint64 {
|
||||
if x != nil {
|
||||
return x.IndexBytes
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *MetadataKey) GetSegments() uint32 {
|
||||
if x != nil {
|
||||
return x.Segments
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type MetadataKeysResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Keys []*MetadataKey `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"`
|
||||
Version uint64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
// The table's rows, and its vector index type (empty without one).
|
||||
Rows uint64 `protobuf:"varint,3,opt,name=rows,proto3" json:"rows,omitempty"`
|
||||
VectorIndex string `protobuf:"bytes,4,opt,name=vector_index,json=vectorIndex,proto3" json:"vector_index,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) Reset() {
|
||||
*x = MetadataKeysResponse{}
|
||||
mi := &file_vector_proto_msgTypes[22]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*MetadataKeysResponse) ProtoMessage() {}
|
||||
|
||||
func (x *MetadataKeysResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_vector_proto_msgTypes[22]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use MetadataKeysResponse.ProtoReflect.Descriptor instead.
|
||||
func (*MetadataKeysResponse) Descriptor() ([]byte, []int) {
|
||||
return file_vector_proto_rawDescGZIP(), []int{22}
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) GetKeys() []*MetadataKey {
|
||||
if x != nil {
|
||||
return x.Keys
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) GetVersion() uint64 {
|
||||
if x != nil {
|
||||
return x.Version
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) GetRows() uint64 {
|
||||
if x != nil {
|
||||
return x.Rows
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *MetadataKeysResponse) GetVectorIndex() string {
|
||||
if x != nil {
|
||||
return x.VectorIndex
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type UpdateMetadataKeyRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Table *Table `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"`
|
||||
Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
|
||||
Remove bool `protobuf:"varint,3,opt,name=remove,proto3" json:"remove,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) Reset() {
|
||||
*x = UpdateMetadataKeyRequest{}
|
||||
mi := &file_vector_proto_msgTypes[23]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UpdateMetadataKeyRequest) ProtoMessage() {}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_vector_proto_msgTypes[23]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UpdateMetadataKeyRequest.ProtoReflect.Descriptor instead.
|
||||
func (*UpdateMetadataKeyRequest) Descriptor() ([]byte, []int) {
|
||||
return file_vector_proto_rawDescGZIP(), []int{23}
|
||||
}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) GetTable() *Table {
|
||||
if x != nil {
|
||||
return x.Table
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) GetKey() string {
|
||||
if x != nil {
|
||||
return x.Key
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UpdateMetadataKeyRequest) GetRemove() bool {
|
||||
if x != nil {
|
||||
return x.Remove
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_vector_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_vector_proto_rawDesc = "" +
|
||||
@@ -1360,13 +1628,33 @@ const file_vector_proto_rawDesc = "" +
|
||||
"\x13FilterStatsResponse\x12-\n" +
|
||||
"\x04keys\x18\x01 \x03(\v2\x19.vector_pb.FilterKeyStatsR\x04keys\x12%\n" +
|
||||
"\x0ewindow_seconds\x18\x02 \x01(\rR\rwindowSeconds\x12\x1c\n" +
|
||||
"\ttruncated\x18\x03 \x01(\bR\ttruncated*V\n" +
|
||||
"\ttruncated\x18\x03 \x01(\bR\ttruncated\"=\n" +
|
||||
"\x13MetadataKeysRequest\x12&\n" +
|
||||
"\x05table\x18\x01 \x01(\v2\x10.vector_pb.TableR\x05table\"\xc1\x01\n" +
|
||||
"\vMetadataKey\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05state\x18\x02 \x01(\tR\x05state\x12\x16\n" +
|
||||
"\x06column\x18\x03 \x01(\tR\x06column\x12!\n" +
|
||||
"\findexed_rows\x18\x04 \x01(\x04R\vindexedRows\x12\x12\n" +
|
||||
"\x04rows\x18\x05 \x01(\x04R\x04rows\x12\x1f\n" +
|
||||
"\vindex_bytes\x18\x06 \x01(\x04R\n" +
|
||||
"indexBytes\x12\x1a\n" +
|
||||
"\bsegments\x18\a \x01(\rR\bsegments\"\x93\x01\n" +
|
||||
"\x14MetadataKeysResponse\x12*\n" +
|
||||
"\x04keys\x18\x01 \x03(\v2\x16.vector_pb.MetadataKeyR\x04keys\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\x04R\aversion\x12\x12\n" +
|
||||
"\x04rows\x18\x03 \x01(\x04R\x04rows\x12!\n" +
|
||||
"\fvector_index\x18\x04 \x01(\tR\vvectorIndex\"l\n" +
|
||||
"\x18UpdateMetadataKeyRequest\x12&\n" +
|
||||
"\x05table\x18\x01 \x01(\v2\x10.vector_pb.TableR\x05table\x12\x10\n" +
|
||||
"\x03key\x18\x02 \x01(\tR\x03key\x12\x16\n" +
|
||||
"\x06remove\x18\x03 \x01(\bR\x06remove*V\n" +
|
||||
"\x0eDistanceMetric\x12\x1f\n" +
|
||||
"\x1bDISTANCE_METRIC_UNSPECIFIED\x10\x00\x12\x06\n" +
|
||||
"\x02L2\x10\x01\x12\n" +
|
||||
"\n" +
|
||||
"\x06COSINE\x10\x02\x12\x0f\n" +
|
||||
"\vDOT_PRODUCT\x10\x032\xfc\x04\n" +
|
||||
"\vDOT_PRODUCT\x10\x032\xa8\x06\n" +
|
||||
"\rVectorService\x12L\n" +
|
||||
"\vCreateIndex\x12\x1d.vector_pb.CreateIndexRequest\x1a\x1e.vector_pb.CreateIndexResponse\x12F\n" +
|
||||
"\tDropIndex\x12\x1b.vector_pb.DropIndexRequest\x1a\x1c.vector_pb.DropIndexResponse\x12I\n" +
|
||||
@@ -1377,7 +1665,9 @@ const file_vector_proto_rawDesc = "" +
|
||||
"\rDeleteVectors\x12\x1f.vector_pb.DeleteVectorsRequest\x1a .vector_pb.DeleteVectorsResponse\x12L\n" +
|
||||
"\vListVectors\x12\x1d.vector_pb.ListVectorsRequest\x1a\x1e.vector_pb.ListVectorsResponse\x12O\n" +
|
||||
"\fQueryVectors\x12\x1e.vector_pb.QueryVectorsRequest\x1a\x1f.vector_pb.QueryVectorsResponse\x12L\n" +
|
||||
"\vFilterStats\x12\x1d.vector_pb.FilterStatsRequest\x1a\x1e.vector_pb.FilterStatsResponseB2Z0github.com/seaweedfs/seaweedfs/weed/pb/vector_pbb\x06proto3"
|
||||
"\vFilterStats\x12\x1d.vector_pb.FilterStatsRequest\x1a\x1e.vector_pb.FilterStatsResponse\x12O\n" +
|
||||
"\fMetadataKeys\x12\x1e.vector_pb.MetadataKeysRequest\x1a\x1f.vector_pb.MetadataKeysResponse\x12Y\n" +
|
||||
"\x11UpdateMetadataKey\x12#.vector_pb.UpdateMetadataKeyRequest\x1a\x1f.vector_pb.MetadataKeysResponseB2Z0github.com/seaweedfs/seaweedfs/weed/pb/vector_pbb\x06proto3"
|
||||
|
||||
var (
|
||||
file_vector_proto_rawDescOnce sync.Once
|
||||
@@ -1392,36 +1682,40 @@ func file_vector_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_vector_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_vector_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
|
||||
var file_vector_proto_msgTypes = make([]protoimpl.MessageInfo, 24)
|
||||
var file_vector_proto_goTypes = []any{
|
||||
(DistanceMetric)(0), // 0: vector_pb.DistanceMetric
|
||||
(*Table)(nil), // 1: vector_pb.Table
|
||||
(*CreateIndexRequest)(nil), // 2: vector_pb.CreateIndexRequest
|
||||
(*CreateIndexResponse)(nil), // 3: vector_pb.CreateIndexResponse
|
||||
(*DropIndexRequest)(nil), // 4: vector_pb.DropIndexRequest
|
||||
(*DropIndexResponse)(nil), // 5: vector_pb.DropIndexResponse
|
||||
(*Vector)(nil), // 6: vector_pb.Vector
|
||||
(*PutVectorsRequest)(nil), // 7: vector_pb.PutVectorsRequest
|
||||
(*PutVectorsResponse)(nil), // 8: vector_pb.PutVectorsResponse
|
||||
(*GetVectorsRequest)(nil), // 9: vector_pb.GetVectorsRequest
|
||||
(*GetVectorsResponse)(nil), // 10: vector_pb.GetVectorsResponse
|
||||
(*DeleteVectorsRequest)(nil), // 11: vector_pb.DeleteVectorsRequest
|
||||
(*DeleteVectorsResponse)(nil), // 12: vector_pb.DeleteVectorsResponse
|
||||
(*ListVectorsRequest)(nil), // 13: vector_pb.ListVectorsRequest
|
||||
(*ListVectorsResponse)(nil), // 14: vector_pb.ListVectorsResponse
|
||||
(*QueryVectorsRequest)(nil), // 15: vector_pb.QueryVectorsRequest
|
||||
(*QueryResult)(nil), // 16: vector_pb.QueryResult
|
||||
(*QueryVectorsResponse)(nil), // 17: vector_pb.QueryVectorsResponse
|
||||
(*FilterStatsRequest)(nil), // 18: vector_pb.FilterStatsRequest
|
||||
(*FilterKeyStats)(nil), // 19: vector_pb.FilterKeyStats
|
||||
(*FilterStatsResponse)(nil), // 20: vector_pb.FilterStatsResponse
|
||||
(*structpb.Struct)(nil), // 21: google.protobuf.Struct
|
||||
(DistanceMetric)(0), // 0: vector_pb.DistanceMetric
|
||||
(*Table)(nil), // 1: vector_pb.Table
|
||||
(*CreateIndexRequest)(nil), // 2: vector_pb.CreateIndexRequest
|
||||
(*CreateIndexResponse)(nil), // 3: vector_pb.CreateIndexResponse
|
||||
(*DropIndexRequest)(nil), // 4: vector_pb.DropIndexRequest
|
||||
(*DropIndexResponse)(nil), // 5: vector_pb.DropIndexResponse
|
||||
(*Vector)(nil), // 6: vector_pb.Vector
|
||||
(*PutVectorsRequest)(nil), // 7: vector_pb.PutVectorsRequest
|
||||
(*PutVectorsResponse)(nil), // 8: vector_pb.PutVectorsResponse
|
||||
(*GetVectorsRequest)(nil), // 9: vector_pb.GetVectorsRequest
|
||||
(*GetVectorsResponse)(nil), // 10: vector_pb.GetVectorsResponse
|
||||
(*DeleteVectorsRequest)(nil), // 11: vector_pb.DeleteVectorsRequest
|
||||
(*DeleteVectorsResponse)(nil), // 12: vector_pb.DeleteVectorsResponse
|
||||
(*ListVectorsRequest)(nil), // 13: vector_pb.ListVectorsRequest
|
||||
(*ListVectorsResponse)(nil), // 14: vector_pb.ListVectorsResponse
|
||||
(*QueryVectorsRequest)(nil), // 15: vector_pb.QueryVectorsRequest
|
||||
(*QueryResult)(nil), // 16: vector_pb.QueryResult
|
||||
(*QueryVectorsResponse)(nil), // 17: vector_pb.QueryVectorsResponse
|
||||
(*FilterStatsRequest)(nil), // 18: vector_pb.FilterStatsRequest
|
||||
(*FilterKeyStats)(nil), // 19: vector_pb.FilterKeyStats
|
||||
(*FilterStatsResponse)(nil), // 20: vector_pb.FilterStatsResponse
|
||||
(*MetadataKeysRequest)(nil), // 21: vector_pb.MetadataKeysRequest
|
||||
(*MetadataKey)(nil), // 22: vector_pb.MetadataKey
|
||||
(*MetadataKeysResponse)(nil), // 23: vector_pb.MetadataKeysResponse
|
||||
(*UpdateMetadataKeyRequest)(nil), // 24: vector_pb.UpdateMetadataKeyRequest
|
||||
(*structpb.Struct)(nil), // 25: google.protobuf.Struct
|
||||
}
|
||||
var file_vector_proto_depIdxs = []int32{
|
||||
1, // 0: vector_pb.CreateIndexRequest.table:type_name -> vector_pb.Table
|
||||
0, // 1: vector_pb.CreateIndexRequest.distance_metric:type_name -> vector_pb.DistanceMetric
|
||||
1, // 2: vector_pb.DropIndexRequest.table:type_name -> vector_pb.Table
|
||||
21, // 3: vector_pb.Vector.metadata:type_name -> google.protobuf.Struct
|
||||
25, // 3: vector_pb.Vector.metadata:type_name -> google.protobuf.Struct
|
||||
1, // 4: vector_pb.PutVectorsRequest.table:type_name -> vector_pb.Table
|
||||
6, // 5: vector_pb.PutVectorsRequest.vectors:type_name -> vector_pb.Vector
|
||||
1, // 6: vector_pb.GetVectorsRequest.table:type_name -> vector_pb.Table
|
||||
@@ -1430,31 +1724,38 @@ var file_vector_proto_depIdxs = []int32{
|
||||
1, // 9: vector_pb.ListVectorsRequest.table:type_name -> vector_pb.Table
|
||||
6, // 10: vector_pb.ListVectorsResponse.vectors:type_name -> vector_pb.Vector
|
||||
1, // 11: vector_pb.QueryVectorsRequest.table:type_name -> vector_pb.Table
|
||||
21, // 12: vector_pb.QueryResult.metadata:type_name -> google.protobuf.Struct
|
||||
25, // 12: vector_pb.QueryResult.metadata:type_name -> google.protobuf.Struct
|
||||
16, // 13: vector_pb.QueryVectorsResponse.vectors:type_name -> vector_pb.QueryResult
|
||||
1, // 14: vector_pb.FilterStatsRequest.table:type_name -> vector_pb.Table
|
||||
19, // 15: vector_pb.FilterStatsResponse.keys:type_name -> vector_pb.FilterKeyStats
|
||||
2, // 16: vector_pb.VectorService.CreateIndex:input_type -> vector_pb.CreateIndexRequest
|
||||
4, // 17: vector_pb.VectorService.DropIndex:input_type -> vector_pb.DropIndexRequest
|
||||
7, // 18: vector_pb.VectorService.PutVectors:input_type -> vector_pb.PutVectorsRequest
|
||||
9, // 19: vector_pb.VectorService.GetVectors:input_type -> vector_pb.GetVectorsRequest
|
||||
11, // 20: vector_pb.VectorService.DeleteVectors:input_type -> vector_pb.DeleteVectorsRequest
|
||||
13, // 21: vector_pb.VectorService.ListVectors:input_type -> vector_pb.ListVectorsRequest
|
||||
15, // 22: vector_pb.VectorService.QueryVectors:input_type -> vector_pb.QueryVectorsRequest
|
||||
18, // 23: vector_pb.VectorService.FilterStats:input_type -> vector_pb.FilterStatsRequest
|
||||
3, // 24: vector_pb.VectorService.CreateIndex:output_type -> vector_pb.CreateIndexResponse
|
||||
5, // 25: vector_pb.VectorService.DropIndex:output_type -> vector_pb.DropIndexResponse
|
||||
8, // 26: vector_pb.VectorService.PutVectors:output_type -> vector_pb.PutVectorsResponse
|
||||
10, // 27: vector_pb.VectorService.GetVectors:output_type -> vector_pb.GetVectorsResponse
|
||||
12, // 28: vector_pb.VectorService.DeleteVectors:output_type -> vector_pb.DeleteVectorsResponse
|
||||
14, // 29: vector_pb.VectorService.ListVectors:output_type -> vector_pb.ListVectorsResponse
|
||||
17, // 30: vector_pb.VectorService.QueryVectors:output_type -> vector_pb.QueryVectorsResponse
|
||||
20, // 31: vector_pb.VectorService.FilterStats:output_type -> vector_pb.FilterStatsResponse
|
||||
24, // [24:32] is the sub-list for method output_type
|
||||
16, // [16:24] is the sub-list for method input_type
|
||||
16, // [16:16] is the sub-list for extension type_name
|
||||
16, // [16:16] is the sub-list for extension extendee
|
||||
0, // [0:16] is the sub-list for field type_name
|
||||
1, // 16: vector_pb.MetadataKeysRequest.table:type_name -> vector_pb.Table
|
||||
22, // 17: vector_pb.MetadataKeysResponse.keys:type_name -> vector_pb.MetadataKey
|
||||
1, // 18: vector_pb.UpdateMetadataKeyRequest.table:type_name -> vector_pb.Table
|
||||
2, // 19: vector_pb.VectorService.CreateIndex:input_type -> vector_pb.CreateIndexRequest
|
||||
4, // 20: vector_pb.VectorService.DropIndex:input_type -> vector_pb.DropIndexRequest
|
||||
7, // 21: vector_pb.VectorService.PutVectors:input_type -> vector_pb.PutVectorsRequest
|
||||
9, // 22: vector_pb.VectorService.GetVectors:input_type -> vector_pb.GetVectorsRequest
|
||||
11, // 23: vector_pb.VectorService.DeleteVectors:input_type -> vector_pb.DeleteVectorsRequest
|
||||
13, // 24: vector_pb.VectorService.ListVectors:input_type -> vector_pb.ListVectorsRequest
|
||||
15, // 25: vector_pb.VectorService.QueryVectors:input_type -> vector_pb.QueryVectorsRequest
|
||||
18, // 26: vector_pb.VectorService.FilterStats:input_type -> vector_pb.FilterStatsRequest
|
||||
21, // 27: vector_pb.VectorService.MetadataKeys:input_type -> vector_pb.MetadataKeysRequest
|
||||
24, // 28: vector_pb.VectorService.UpdateMetadataKey:input_type -> vector_pb.UpdateMetadataKeyRequest
|
||||
3, // 29: vector_pb.VectorService.CreateIndex:output_type -> vector_pb.CreateIndexResponse
|
||||
5, // 30: vector_pb.VectorService.DropIndex:output_type -> vector_pb.DropIndexResponse
|
||||
8, // 31: vector_pb.VectorService.PutVectors:output_type -> vector_pb.PutVectorsResponse
|
||||
10, // 32: vector_pb.VectorService.GetVectors:output_type -> vector_pb.GetVectorsResponse
|
||||
12, // 33: vector_pb.VectorService.DeleteVectors:output_type -> vector_pb.DeleteVectorsResponse
|
||||
14, // 34: vector_pb.VectorService.ListVectors:output_type -> vector_pb.ListVectorsResponse
|
||||
17, // 35: vector_pb.VectorService.QueryVectors:output_type -> vector_pb.QueryVectorsResponse
|
||||
20, // 36: vector_pb.VectorService.FilterStats:output_type -> vector_pb.FilterStatsResponse
|
||||
23, // 37: vector_pb.VectorService.MetadataKeys:output_type -> vector_pb.MetadataKeysResponse
|
||||
23, // 38: vector_pb.VectorService.UpdateMetadataKey:output_type -> vector_pb.MetadataKeysResponse
|
||||
29, // [29:39] is the sub-list for method output_type
|
||||
19, // [19:29] is the sub-list for method input_type
|
||||
19, // [19:19] is the sub-list for extension type_name
|
||||
19, // [19:19] is the sub-list for extension extendee
|
||||
0, // [0:19] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_vector_proto_init() }
|
||||
@@ -1468,7 +1769,7 @@ func file_vector_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_vector_proto_rawDesc), len(file_vector_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 20,
|
||||
NumMessages: 24,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -19,14 +19,16 @@ import (
|
||||
const _ = grpc.SupportPackageIsVersion9
|
||||
|
||||
const (
|
||||
VectorService_CreateIndex_FullMethodName = "/vector_pb.VectorService/CreateIndex"
|
||||
VectorService_DropIndex_FullMethodName = "/vector_pb.VectorService/DropIndex"
|
||||
VectorService_PutVectors_FullMethodName = "/vector_pb.VectorService/PutVectors"
|
||||
VectorService_GetVectors_FullMethodName = "/vector_pb.VectorService/GetVectors"
|
||||
VectorService_DeleteVectors_FullMethodName = "/vector_pb.VectorService/DeleteVectors"
|
||||
VectorService_ListVectors_FullMethodName = "/vector_pb.VectorService/ListVectors"
|
||||
VectorService_QueryVectors_FullMethodName = "/vector_pb.VectorService/QueryVectors"
|
||||
VectorService_FilterStats_FullMethodName = "/vector_pb.VectorService/FilterStats"
|
||||
VectorService_CreateIndex_FullMethodName = "/vector_pb.VectorService/CreateIndex"
|
||||
VectorService_DropIndex_FullMethodName = "/vector_pb.VectorService/DropIndex"
|
||||
VectorService_PutVectors_FullMethodName = "/vector_pb.VectorService/PutVectors"
|
||||
VectorService_GetVectors_FullMethodName = "/vector_pb.VectorService/GetVectors"
|
||||
VectorService_DeleteVectors_FullMethodName = "/vector_pb.VectorService/DeleteVectors"
|
||||
VectorService_ListVectors_FullMethodName = "/vector_pb.VectorService/ListVectors"
|
||||
VectorService_QueryVectors_FullMethodName = "/vector_pb.VectorService/QueryVectors"
|
||||
VectorService_FilterStats_FullMethodName = "/vector_pb.VectorService/FilterStats"
|
||||
VectorService_MetadataKeys_FullMethodName = "/vector_pb.VectorService/MetadataKeys"
|
||||
VectorService_UpdateMetadataKey_FullMethodName = "/vector_pb.VectorService/UpdateMetadataKey"
|
||||
)
|
||||
|
||||
// VectorServiceClient is the client API for VectorService service.
|
||||
@@ -47,6 +49,11 @@ type VectorServiceClient interface {
|
||||
// How this engine's queries of an index filtered on each metadata key
|
||||
// over the last hour.
|
||||
FilterStats(ctx context.Context, in *FilterStatsRequest, opts ...grpc.CallOption) (*FilterStatsResponse, error)
|
||||
// The index's indexed metadata keys, as its Lance table records them.
|
||||
MetadataKeys(ctx context.Context, in *MetadataKeysRequest, opts ...grpc.CallOption) (*MetadataKeysResponse, error)
|
||||
// Ask for a metadata key to be indexed, or no longer: the table records
|
||||
// the request and the maintenance worker carries it out.
|
||||
UpdateMetadataKey(ctx context.Context, in *UpdateMetadataKeyRequest, opts ...grpc.CallOption) (*MetadataKeysResponse, error)
|
||||
}
|
||||
|
||||
type vectorServiceClient struct {
|
||||
@@ -137,6 +144,26 @@ func (c *vectorServiceClient) FilterStats(ctx context.Context, in *FilterStatsRe
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *vectorServiceClient) MetadataKeys(ctx context.Context, in *MetadataKeysRequest, opts ...grpc.CallOption) (*MetadataKeysResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(MetadataKeysResponse)
|
||||
err := c.cc.Invoke(ctx, VectorService_MetadataKeys_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *vectorServiceClient) UpdateMetadataKey(ctx context.Context, in *UpdateMetadataKeyRequest, opts ...grpc.CallOption) (*MetadataKeysResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(MetadataKeysResponse)
|
||||
err := c.cc.Invoke(ctx, VectorService_UpdateMetadataKey_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VectorServiceServer is the server API for VectorService service.
|
||||
// All implementations must embed UnimplementedVectorServiceServer
|
||||
// for forward compatibility.
|
||||
@@ -155,6 +182,11 @@ type VectorServiceServer interface {
|
||||
// How this engine's queries of an index filtered on each metadata key
|
||||
// over the last hour.
|
||||
FilterStats(context.Context, *FilterStatsRequest) (*FilterStatsResponse, error)
|
||||
// The index's indexed metadata keys, as its Lance table records them.
|
||||
MetadataKeys(context.Context, *MetadataKeysRequest) (*MetadataKeysResponse, error)
|
||||
// Ask for a metadata key to be indexed, or no longer: the table records
|
||||
// the request and the maintenance worker carries it out.
|
||||
UpdateMetadataKey(context.Context, *UpdateMetadataKeyRequest) (*MetadataKeysResponse, error)
|
||||
mustEmbedUnimplementedVectorServiceServer()
|
||||
}
|
||||
|
||||
@@ -189,6 +221,12 @@ func (UnimplementedVectorServiceServer) QueryVectors(context.Context, *QueryVect
|
||||
func (UnimplementedVectorServiceServer) FilterStats(context.Context, *FilterStatsRequest) (*FilterStatsResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method FilterStats not implemented")
|
||||
}
|
||||
func (UnimplementedVectorServiceServer) MetadataKeys(context.Context, *MetadataKeysRequest) (*MetadataKeysResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method MetadataKeys not implemented")
|
||||
}
|
||||
func (UnimplementedVectorServiceServer) UpdateMetadataKey(context.Context, *UpdateMetadataKeyRequest) (*MetadataKeysResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method UpdateMetadataKey not implemented")
|
||||
}
|
||||
func (UnimplementedVectorServiceServer) mustEmbedUnimplementedVectorServiceServer() {}
|
||||
func (UnimplementedVectorServiceServer) testEmbeddedByValue() {}
|
||||
|
||||
@@ -354,6 +392,42 @@ func _VectorService_FilterStats_Handler(srv interface{}, ctx context.Context, de
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VectorService_MetadataKeys_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(MetadataKeysRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VectorServiceServer).MetadataKeys(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VectorService_MetadataKeys_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VectorServiceServer).MetadataKeys(ctx, req.(*MetadataKeysRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VectorService_UpdateMetadataKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(UpdateMetadataKeyRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VectorServiceServer).UpdateMetadataKey(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VectorService_UpdateMetadataKey_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VectorServiceServer).UpdateMetadataKey(ctx, req.(*UpdateMetadataKeyRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// VectorService_ServiceDesc is the grpc.ServiceDesc for VectorService service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
@@ -393,6 +467,14 @@ var VectorService_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "FilterStats",
|
||||
Handler: _VectorService_FilterStats_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "MetadataKeys",
|
||||
Handler: _VectorService_MetadataKeys_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "UpdateMetadataKey",
|
||||
Handler: _VectorService_UpdateMetadataKey_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
Metadata: "vector.proto",
|
||||
|
||||
@@ -122,3 +122,17 @@ func (c *describedClient) QueryVectors(ctx context.Context, in *vector_pb.QueryV
|
||||
}
|
||||
return c.VectorServiceClient.QueryVectors(ctx, in, opts...)
|
||||
}
|
||||
|
||||
func (c *describedClient) MetadataKeys(ctx context.Context, in *vector_pb.MetadataKeysRequest, opts ...grpc.CallOption) (*vector_pb.MetadataKeysResponse, error) {
|
||||
if err := c.resolve(ctx, in.Table); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.VectorServiceClient.MetadataKeys(ctx, in, opts...)
|
||||
}
|
||||
|
||||
func (c *describedClient) UpdateMetadataKey(ctx context.Context, in *vector_pb.UpdateMetadataKeyRequest, opts ...grpc.CallOption) (*vector_pb.MetadataKeysResponse, error) {
|
||||
if err := c.resolve(ctx, in.Table); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.VectorServiceClient.UpdateMetadataKey(ctx, in, opts...)
|
||||
}
|
||||
@@ -347,6 +347,48 @@ func (d *vectorDiscovery) FilterStats(ctx context.Context, in *vector_pb.FilterS
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (d *vectorDiscovery) MetadataKeys(ctx context.Context, in *vector_pb.MetadataKeysRequest, opts ...grpc.CallOption) (resp *vector_pb.MetadataKeysResponse, err error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
var c vector_pb.VectorServiceClient
|
||||
if c, err = d.cli(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err = c.MetadataKeys(ctx, in, opts...)
|
||||
if isTransportError(err) {
|
||||
d.reset(c)
|
||||
}
|
||||
d.release(c)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !isTransportError(err) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (d *vectorDiscovery) UpdateMetadataKey(ctx context.Context, in *vector_pb.UpdateMetadataKeyRequest, opts ...grpc.CallOption) (resp *vector_pb.MetadataKeysResponse, err error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
var c vector_pb.VectorServiceClient
|
||||
if c, err = d.cli(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err = c.UpdateMetadataKey(ctx, in, opts...)
|
||||
if isTransportError(err) {
|
||||
d.reset(c)
|
||||
}
|
||||
d.release(c)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !isTransportError(err) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func discoverVectorServers(masters []pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string) []string {
|
||||
for _, master := range masters {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package s3vectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
// MetadataColumnsCapability is the gRPC health service an engine reports
|
||||
// serving when its writes fill indexed metadata key columns.
|
||||
const MetadataColumnsCapability = "weed.vector.capability.metadata_columns"
|
||||
|
||||
// MetadataKeys lists the indexed metadata keys of an index, as its table
|
||||
// records them. The caller is trusted: the shell and the admin server.
|
||||
func (s *Server) MetadataKeys(ctx context.Context, bucket, index string) (*vector_pb.MetadataKeysResponse, error) {
|
||||
if _, err := s.trustedIndex(ctx, bucket, index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.hasEngine() {
|
||||
return nil, errNoEngine
|
||||
}
|
||||
return s.vectors.MetadataKeys(ctx, &vector_pb.MetadataKeysRequest{Table: tableID(bucket, index)})
|
||||
}
|
||||
|
||||
// UpdateMetadataKey asks for a metadata key of an index to be indexed, or no
|
||||
// longer. Adding refuses a non-filterable key, and refuses while a registered
|
||||
// engine does not fill indexed key columns on its writes.
|
||||
func (s *Server) UpdateMetadataKey(ctx context.Context, bucket, index, key string, remove bool) (*vector_pb.MetadataKeysResponse, error) {
|
||||
idx, err := s.trustedIndex(ctx, bucket, index)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !s.hasEngine() {
|
||||
return nil, errNoEngine
|
||||
}
|
||||
if !remove {
|
||||
if err := metadataKeyIndexable(idx, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.checkMetadataColumns(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return s.vectors.UpdateMetadataKey(ctx, &vector_pb.UpdateMetadataKeyRequest{
|
||||
Table: tableID(bucket, index),
|
||||
Key: key,
|
||||
Remove: remove,
|
||||
})
|
||||
}
|
||||
|
||||
var errNoEngine = fmt.Errorf("no vector engine is configured")
|
||||
|
||||
// hasEngine reports whether calls can reach an engine client.
|
||||
func (s *Server) hasEngine() bool {
|
||||
backend := s.vectors
|
||||
if described, ok := backend.(*describedClient); ok {
|
||||
backend = described.VectorServiceClient
|
||||
}
|
||||
return backend != nil
|
||||
}
|
||||
|
||||
func (s *Server) trustedIndex(ctx context.Context, bucket, index string) (*Index, error) {
|
||||
var record map[string]interface{}
|
||||
if err := s.internalCatalog(ctx, "GetTable", &s3tables.GetTableRequest{
|
||||
TableBucketARN: s.s3tablesArn(bucket),
|
||||
Namespace: []string{indexNamespace},
|
||||
Name: index,
|
||||
}, &record); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return indexFromRecord(s, bucket, index, record), nil
|
||||
}
|
||||
|
||||
func metadataKeyIndexable(idx *Index, key string) error {
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return fmt.Errorf("the metadata key is empty")
|
||||
}
|
||||
if idx.MetadataConfiguration != nil && slices.Contains(idx.MetadataConfiguration.NonFilterableMetadataKeys, key) {
|
||||
return fmt.Errorf("%s is a non-filterable metadata key of index %s; only filterable keys can be indexed", key, idx.IndexName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMetadataColumns refuses unless every registered engine reports the
|
||||
// capability, when the engines come from master discovery.
|
||||
func (s *Server) checkMetadataColumns(ctx context.Context) error {
|
||||
backend := s.vectors
|
||||
if described, ok := backend.(*describedClient); ok {
|
||||
backend = described.VectorServiceClient
|
||||
}
|
||||
if d, ok := backend.(*vectorDiscovery); ok {
|
||||
return d.CheckMetadataColumns(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckMetadataColumns returns an error unless engines are registered and
|
||||
// each reports MetadataColumnsCapability.
|
||||
func (d *vectorDiscovery) CheckMetadataColumns(ctx context.Context) error {
|
||||
return checkMetadataColumns(ctx, d.discover(d.masters(), d.grpcDialOption), d.grpcDialOption)
|
||||
}
|
||||
|
||||
func checkMetadataColumns(ctx context.Context, engines []string, dialOption grpc.DialOption) error {
|
||||
if len(engines) == 0 {
|
||||
return fmt.Errorf("no vector engine is registered")
|
||||
}
|
||||
var lacking []string
|
||||
for _, address := range engines {
|
||||
if err := engineReports(ctx, address, dialOption, MetadataColumnsCapability); err != nil {
|
||||
lacking = append(lacking, fmt.Sprintf("%s (%v)", address, err))
|
||||
}
|
||||
}
|
||||
if len(lacking) > 0 {
|
||||
slices.Sort(lacking)
|
||||
return fmt.Errorf("vector engines that do not fill indexed metadata key columns are registered; upgrade them first: %s", strings.Join(lacking, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func engineReports(ctx context.Context, address string, dialOption grpc.DialOption, service string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
conn, err := pb.GrpcDial(ctx, address, false, dialOption)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
resp, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{Service: service})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING {
|
||||
return fmt.Errorf("%s is %s", service, resp.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package s3vectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/health"
|
||||
"google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
func healthServer(t *testing.T, services ...string) string {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := grpc.NewServer()
|
||||
h := health.NewServer()
|
||||
for _, service := range services {
|
||||
h.SetServingStatus(service, grpc_health_v1.HealthCheckResponse_SERVING)
|
||||
}
|
||||
grpc_health_v1.RegisterHealthServer(server, h)
|
||||
go server.Serve(listener)
|
||||
t.Cleanup(server.Stop)
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
func TestCheckMetadataColumnsNeedsEveryEngine(t *testing.T) {
|
||||
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
capable := healthServer(t, "vector_pb.VectorService", MetadataColumnsCapability)
|
||||
old := healthServer(t, "vector_pb.VectorService")
|
||||
ctx := context.Background()
|
||||
if err := checkMetadataColumns(ctx, []string{capable}, dial); err != nil {
|
||||
t.Fatalf("a capable engine: %v", err)
|
||||
}
|
||||
err := checkMetadataColumns(ctx, []string{capable, old}, dial)
|
||||
if err == nil || !strings.Contains(err.Error(), old) || strings.Contains(err.Error(), capable+" ") {
|
||||
t.Fatalf("an old engine must be named: %v", err)
|
||||
}
|
||||
if err := checkMetadataColumns(ctx, nil, dial); err == nil {
|
||||
t.Fatal("no engine must refuse")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataKeyIndexable(t *testing.T) {
|
||||
idx := &Index{IndexName: "i", MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}}
|
||||
if err := metadataKeyIndexable(idx, "author"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"body", "", " "} {
|
||||
if metadataKeyIndexable(idx, key) == nil {
|
||||
t.Fatalf("%q must be refused", key)
|
||||
}
|
||||
}
|
||||
if err := metadataKeyIndexable(&Index{}, "author"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataKeysWithoutAnEngine(t *testing.T) {
|
||||
s := &Server{}
|
||||
if s.hasEngine() {
|
||||
t.Fatal("a server without a client has an engine")
|
||||
}
|
||||
s.vectors = &describedClient{}
|
||||
if s.hasEngine() {
|
||||
t.Fatal("a describe wrapper around no client has an engine")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user