mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-11 14:28:33 +01:00
Chris Lu
8236c8d136
* vector: expose request metrics and registration-aware health endpoints * vector: report immutable read cache and storage fetch metrics * docs: describe vector engine health, metrics, and maintenance * vector: count canceled requests when their handler is dropped * vector: account for uncached large immutable storage reads * vector: reliably release the stalled credential reader in tests * vector: observe large reads as their streamed body is consumed * vector: bound heartbeats on established master registration streams * test: exercise hardened vector maintenance on Go and Rust volumes * vector: register engines in their configured filer group * vector: make retried index creation preserve compatible datasets * vector: allow registration to advertise a separate address * test: release stalled credential readers without blocking FIFO cleanup * test: allow hardened vector coverage in a named filer group * vector: discover engines in the same filer group as their gateway * vector: fall back when the advertised address is empty * admin: expose vector readiness, maintenance, and management tools (#12520) * s3vectors: share vector discovery with trusted management clients * admin: show vector engine readiness and recent maintenance activity * admin: suggest S3 Vectors IAM actions from the supported operations * admin: manage vector indexes, policies, tags, encryption, and query data * admin: validate SSE-S3 support before creating encrypted vector buckets * admin: discover vector engines in their registration group * admin: probe vector engines concurrently with independent time budgets * admin: verify encryption capability on the remote S3 gateway * admin: defer discovery until after the HTTP server can start * admin: distinguish unknown vector health services from failed engines * admin: count workers that can execute vector index maintenance * admin: allow read-only users to query vectors and inspect tags * vector: scope gateway and admin discovery to the configured filer group * admin: refresh policy state after mutations and serialize policy controls * vector: drain active RPCs before closing replaced discovery connections * vector: respect request cancellation while waiting for discovery * vector: prefer another registered engine after a transport failure * admin: check all masters before reporting no registered vector engines * iam: honor attached vector policies without broadening legacy grants * vector: allow short-lived discovery clients to close their connections * vector: refresh cached master addresses for each discovery lookup * iam: enforce vector bucket tag conditions in attached policies * iam: authorize vector bucket creation with the requested tags * test: exercise vector IAM grants and tag denies on Go and Rust volumes * admin: reject authenticated vector mutations without an admin role * test: run vector IAM assertions before the shared engine is stopped * vector: add native mini startup, shell tools, and a runnable Compose example (#12522) * s3vectors: share vector discovery with trusted management clients * admin: show vector engine readiness and recent maintenance activity * admin: suggest S3 Vectors IAM actions from the supported operations * admin: manage vector indexes, policies, tags, encryption, and query data * admin: validate SSE-S3 support before creating encrypted vector buckets * admin: discover vector engines in their registration group * admin: probe vector engines concurrently with independent time budgets * admin: verify encryption capability on the remote S3 gateway * admin: defer discovery until after the HTTP server can start * admin: distinguish unknown vector health services from failed engines * admin: count workers that can execute vector index maintenance * admin: allow read-only users to query vectors and inspect tags * vector: scope gateway and admin discovery to the configured filer group * admin: refresh policy state after mutations and serialize policy controls * vector: drain active RPCs before closing replaced discovery connections * vector: respect request cancellation while waiting for discovery * vector: prefer another registered engine after a transport failure * admin: check all masters before reporting no registered vector engines * iam: honor attached vector policies without broadening legacy grants * vector: allow short-lived discovery clients to close their connections * vector: refresh cached master addresses for each discovery lookup * iam: enforce vector bucket tag conditions in attached policies * iam: authorize vector bucket creation with the requested tags * test: exercise vector IAM grants and tag denies on Go and Rust volumes * admin: reject authenticated vector mutations without an admin role * test: run vector IAM assertions before the shared engine is stopped * mini: optionally start and supervise the vector engine * mini: bootstrap vector buckets with existing Lance catalog helpers * shell: manage vector buckets, indexes, and vector data * docs: provide a runnable vector and maintenance worker Compose stack * shell: scope vector discovery to the session and close its connections * mini: honor the bind address for vector and metrics listeners * mini: reserve vector ports before assigning gRPC and compatibility listeners * mini: wait for vector storage setup and master registration * mini: create requested vector buckets without requiring the Lance HTTP listener * shell: refresh master addresses during vector discovery * mini: resolve sibling vector executables with platform extensions * vector: preserve uncertain creates and fix mini listener allocation (#12525) * vector: retain catalog records when index creation has an unknown outcome * mini: keep chosen listener ports reserved throughout allocation * mini: resolve hostname binds before starting vector listeners * vector: fix integration test imports and recover interrupted index creation (#12533) * shell: register vector operations from the command package * s3vectors: resume interrupted index initialization safely * mini: exclude disabled WebDAV from listener allocation * s3vectors: retain uncertain creates across refused retries * s3vectors: claim pending indexes before retrying initialization * s3vectors: compare resolved distance metrics during recovery * s3vectors: preserve metadata keys when retrying initialization * s3vectors: verify ownership before initializing new catalog records
99 lines
4.2 KiB
Go
99 lines
4.2 KiB
Go
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestS3VectorsBucketsRendersBuckets(t *testing.T) {
|
|
data := dash.S3VectorsBucketsData{TotalBuckets: 2, Buckets: []dash.S3VectorsBucketSummary{
|
|
{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media", CreatedAt: time.Unix(0, 0)},
|
|
{Name: "sealed", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/sealed", Encryption: "AES256", CreatedAt: time.Unix(0, 0)},
|
|
}}
|
|
var out bytes.Buffer
|
|
require.NoError(t, S3VectorsBuckets(data).Render(dash.WithURLPrefix(t.Context(), "/admin"), &out))
|
|
page := out.String()
|
|
require.Contains(t, page, `href="/admin/object-store/vector-buckets/media"`)
|
|
require.Contains(t, page, `data-bucket-name="sealed"`)
|
|
require.Contains(t, page, `<span class="badge bg-success">AES256</span>`)
|
|
require.Contains(t, page, "2 vector buckets")
|
|
require.NotContains(t, page, "No vector buckets found")
|
|
require.NotContains(t, page, "showing the first")
|
|
|
|
data.Truncated = true
|
|
out.Reset()
|
|
require.NoError(t, S3VectorsBuckets(data).Render(t.Context(), &out))
|
|
require.Contains(t, out.String(), "showing the first 2; list the rest with an S3 Vectors client")
|
|
}
|
|
|
|
func TestS3VectorsBucketDetailsRendersIndexesAndPolicy(t *testing.T) {
|
|
data := dash.S3VectorsBucketData{
|
|
Bucket: dash.S3VectorsBucketSummary{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"},
|
|
Policy: `{"Version":"2012-10-17","Statement":[]}`,
|
|
TotalIndexes: 1,
|
|
Indexes: []*s3vectors.Index{{
|
|
IndexName: "movies", IndexArn: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media/index/movies",
|
|
DataType: "float32", Dimension: 384, DistanceMetric: "cosine",
|
|
MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"summary", "body"}},
|
|
}},
|
|
}
|
|
var out bytes.Buffer
|
|
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
|
page := out.String()
|
|
for _, want := range []string{"movies", ">384<", ">cosine<", ">float32<", "summary, body", `data-index-name="movies"`, "{\n "Version": "2012-10-17",\n "Statement": []\n}"} {
|
|
require.Contains(t, page, want)
|
|
}
|
|
require.False(t, strings.Contains(page, "No indexes."), "an index is listed")
|
|
|
|
data.Policy = "not json"
|
|
out.Reset()
|
|
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
|
require.Contains(t, out.String(), ">not json</pre>")
|
|
|
|
data.Policy = `{"Version":"2012-10-17","Extra":` + strings.Repeat("[", 5000) + strings.Repeat("]", 5000) + "}"
|
|
out.Reset()
|
|
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
|
require.Contains(t, out.String(), `"Extra":[[[[`, "a deeply nested policy is shown as stored")
|
|
}
|
|
|
|
func TestJSONDepthSkipsStrings(t *testing.T) {
|
|
require.Equal(t, 3, jsonDepth(`{"a":["[{\"[",{}]}`))
|
|
require.Equal(t, 0, jsonDepth(`"[[["`))
|
|
}
|
|
|
|
func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) {
|
|
data := dash.S3TablesBucketsData{Buckets: []dash.S3TablesBucketSummary{
|
|
{Name: "vectors", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/vectors", Format: "LANCE"},
|
|
{Name: "warehouse", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse", Format: "ICEBERG"},
|
|
}}
|
|
var out bytes.Buffer
|
|
require.NoError(t, S3TablesBuckets(data).Render(t.Context(), &out))
|
|
page := out.String()
|
|
require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`)
|
|
require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`)
|
|
}
|
|
|
|
func TestVectorOperationalStates(t *testing.T) {
|
|
for _, data := range []dash.VectorOperationsData{
|
|
{}, {DiscoveryError: "master unavailable"},
|
|
{Engines: []dash.VectorNode{{Address: "engine:8090", Status: "Ready"}}, MaintenanceWorkers: 1},
|
|
} {
|
|
var out bytes.Buffer
|
|
require.NoError(t, VectorStatus(data).Render(t.Context(), &out))
|
|
if data.DiscoveryError != "" {
|
|
require.Contains(t, out.String(), "Engine discovery failed")
|
|
} else if len(data.Engines) == 0 {
|
|
require.Contains(t, out.String(), "No vector engine is registered")
|
|
} else {
|
|
require.Contains(t, out.String(), "engine:8090")
|
|
require.Contains(t, out.String(), "Ready")
|
|
}
|
|
}
|
|
}
|