From 489ebb7dd4674fe3f106f914484d34955fb4654a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 21 Aug 2026 12:33:49 -0700 Subject: [PATCH] Add Spark Lance Integration Spark reaches the Lance catalog through the connector's DSV2 catalog, the same way it reaches the Iceberg one. Covers the storage.* prefix, which is object_store's key names rather than Spark's s3a ones, and the two things the integration suite settled: CREATE TABLE works because the connector declares and writes rather than pushing Arrow at the server, and SHOW TABLES returns namespace identifiers rather than bare table names. --- LanceDB-Integration.md | 1 + SeaweedFS-Lance-Catalog.md | 4 +- Spark-Lance-Integration.md | 106 +++++++++++++++++++++++++++++++++++++ _Sidebar.md | 1 + 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 Spark-Lance-Integration.md diff --git a/LanceDB-Integration.md b/LanceDB-Integration.md index 2180984..8b829e3 100644 --- a/LanceDB-Integration.md +++ b/LanceDB-Integration.md @@ -107,4 +107,5 @@ lance.dataset("s3://vectors/ml/embeddings", storage_options=opts) ## See also - [[SeaweedFS Lance Catalog]] +- [[Spark Lance Integration]] - [[Lance Maintenance Worker]] — compaction, index optimization, version cleanup diff --git a/SeaweedFS-Lance-Catalog.md b/SeaweedFS-Lance-Catalog.md index 77a35ed..4c0730d 100644 --- a/SeaweedFS-Lance-Catalog.md +++ b/SeaweedFS-Lance-Catalog.md @@ -58,7 +58,8 @@ declared = ns.declare_table(ln.DeclareTableRequest(id=["vectors", "ml", "embeddi lance.write_dataset(table, declared.location, storage_options=declared.storage_options) ``` -See [[LanceDB Integration]] for the same thing through LanceDB. +See [[LanceDB Integration]] and [[Spark Lance Integration]] for the same thing +through those clients. ## What the catalog serves @@ -116,6 +117,7 @@ Table buckets show the format they hold and the endpoint that serves them. For a ## See also - [[LanceDB Integration]] +- [[Spark Lance Integration]] - [[Lance Maintenance Worker]] - [[S3 Table Bucket]] - [[SeaweedFS Iceberg Catalog]] diff --git a/Spark-Lance-Integration.md b/Spark-Lance-Integration.md new file mode 100644 index 0000000..39e0526 --- /dev/null +++ b/Spark-Lance-Integration.md @@ -0,0 +1,106 @@ +# Spark Lance Integration + +Spark reads and writes Lance tables in a SeaweedFS table bucket through the +[Lance Spark connector](https://lance.org/integrations/spark/config/), pointed at +the [[SeaweedFS Lance Catalog]]. + +Verified against Spark 3.5.1 and `org.lance:lance-spark-bundle-3.5_2.12:0.7.1` by +the integration suite in `test/s3tables/catalog_spark_lance/`. + +## Prerequisites + +- SeaweedFS with the Lance Namespace enabled: `weed server -s3 -s3.port.lance=9101` +- A table bucket declared `LANCE`: + + ```bash + weed shell + > s3tables.bucket -create -name vectors -format LANCE -account 000000000000 + ``` + +## Configuration + +### PySpark + +```python +from pyspark.sql import SparkSession + +spark = (SparkSession.builder + .appName("SeaweedFS Lance") + .config("spark.jars.packages", "org.lance:lance-spark-bundle-3.5_2.12:0.7.1") + .config("spark.sql.catalog.lance", "org.lance.spark.LanceNamespaceSparkCatalog") + .config("spark.sql.catalog.lance.impl", "rest") + .config("spark.sql.catalog.lance.uri", "http://localhost:9101") + .config("spark.sql.catalog.lance.storage.aws_endpoint", "http://localhost:8333") + .config("spark.sql.catalog.lance.storage.allow_http", "true") + .config("spark.sql.catalog.lance.storage.aws_access_key_id", "…") + .config("spark.sql.catalog.lance.storage.aws_secret_access_key", "…") + .config("spark.sql.catalog.lance.storage.aws_region", "us-east-1") + .getOrCreate()) +``` + +Pick the bundle that matches your Spark and Scala version — `lance-spark-bundle-3.4_2.12`, +`-3.5_2.12`, `-3.5_2.13`, `-4.0_2.13` and others are published under `org.lance`. + +### The `storage.` prefix + +Everything under `spark.sql.catalog..storage.` is handed to lance as +object_store options, so these are **object_store's key names** +(`aws_endpoint`, `aws_access_key_id`, `allow_http`) rather than Spark's `s3a` +ones. Getting this wrong is the usual reason a table lists but will not read. + +Credentials belong there because a gateway without STS configured vends an +endpoint and a region but no credentials of its own. With +`-s3.iceberg.credentialRole` set, the namespace vends real ones and the client +does not need static keys — see [[SeaweedFS Lance Catalog]]. + +## Example SQL + +A table bucket is the first level of the namespace path, so a table is addressed +as `...`: + +```sql +CREATE NAMESPACE IF NOT EXISTS lance.`vectors`.ml; + +CREATE TABLE lance.`vectors`.ml.embeddings ( + id BIGINT, + title STRING, + vector ARRAY +) USING lance; + +INSERT INTO lance.`vectors`.ml.embeddings VALUES + (1, 'one', array(1.0f, 2.0f, 3.0f)), + (2, 'two', array(2.0f, 3.0f, 4.0f)); + +SELECT count(*) FROM lance.`vectors`.ml.embeddings; +SELECT id, title FROM lance.`vectors`.ml.embeddings WHERE id >= 2 ORDER BY id; +``` + +Backticks around the bucket name are needed when it contains a hyphen. + +## What to expect + +- **`CREATE TABLE` works.** The connector declares the table through the catalog + and writes the data itself; it does not push Arrow data at the server, which + is the operation SeaweedFS answers `Unsupported` for. +- **`SHOW TABLES` returns namespace identifiers**, not bare Spark names: + + ``` + SHOW TABLES IN lance.`vectors`.ml + -> vectors$ml$embeddings + ``` + +- **Repeated writes are fine.** A second `INSERT` is the one that fails on a + store that cannot order commits; SeaweedFS evaluates Lance's conditional PUT + atomically at the object's owner filer, so it does not. + +## Without Spark + +The same tables open in [[LanceDB Integration]], in pylance, or straight off +their location with no catalog at all. + +## See also + +- [[SeaweedFS Lance Catalog]] +- [[LanceDB Integration]] +- [[Lance Maintenance Worker]] +- [[Spark Iceberg Integration]] diff --git a/_Sidebar.md b/_Sidebar.md index 4d324f5..1f3dfcb 100644 --- a/_Sidebar.md +++ b/_Sidebar.md @@ -126,6 +126,7 @@ ### Lance Integrations * [[LanceDB Integration]] +* [[Spark Lance Integration]] ### S3 Authentication & IAM * [[S3 Configuration]] - Start Here