From 299bcc30158698f71b6931d214d71143e988cd79 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 15 Sep 2025 22:24:39 -0700 Subject: [PATCH] SMQ --- Kafka-to-Kafka-Gateway-to-SMQ-to-SQL.md | 54 ++++++++++++ PostgreSQL-compatible-Server-weed-db.md | 106 +++++++++++++++++++++++ Pub-Sub-to-SMQ-to-SQL.md | 67 ++++++++++++++ Structured-Data-Lake-with-SMQ-and-SQL.md | 52 +++++++++++ _Sidebar.md | 4 + 5 files changed, 283 insertions(+) create mode 100644 Kafka-to-Kafka-Gateway-to-SMQ-to-SQL.md create mode 100644 PostgreSQL-compatible-Server-weed-db.md create mode 100644 Pub-Sub-to-SMQ-to-SQL.md create mode 100644 Structured-Data-Lake-with-SMQ-and-SQL.md diff --git a/Kafka-to-Kafka-Gateway-to-SMQ-to-SQL.md b/Kafka-to-Kafka-Gateway-to-SMQ-to-SQL.md new file mode 100644 index 0000000..b24897c --- /dev/null +++ b/Kafka-to-Kafka-Gateway-to-SMQ-to-SQL.md @@ -0,0 +1,54 @@ +# Kafka client → Kafka gateway → SMQ → SQL + +Bring your existing Kafka clients. Point them at the SeaweedFS Kafka gateway. Messages flow into Seaweed Message Queue (SMQ) for streaming, while SeaweedFS persists them into Parquet for SQL analytics. + +> See the end-to-end picture: [[Structured Data Lake with SMQ and SQL]]. + +## Why use the Kafka gateway + +- Keep your Kafka tooling and clients +- Scale stateless brokers and storage independently +- Get streaming + Parquet-based analytics without changing producers + +## Architecture + +``` +Kafka Clients <=> SeaweedFS Kafka Gateway <=> SMQ Brokers => Subscribers + \ + +--> SeaweedFS (Parquet) => SQL Engines +``` + +The gateway speaks the Kafka protocol to clients and maps topics/partitions, offsets, and consumer groups to SMQ semantics. + +## What stays the same + +- Kafka client libraries and tooling (producers/consumers) +- Topic/partition concepts +- Consumer groups and offsets + +## What you gain + +- Durable Parquet storage for batch analytics +- One pipeline for both streaming and SQL +- Simple, scalable operations (stateless brokers, disaggregated storage) + +## Getting started + +1) Start SMQ and the Kafka gateway (example ports): + +```bash +weed mq.broker -port=17777 -master=localhost:9333 +weed mq.agent -port=16777 -broker=localhost:17777 +weed mq.kafka -port=19092 -broker=localhost:17777 +``` + +2) Point your Kafka producer/consumer at `localhost:19092`. + +3) Query the resulting Parquet data with your SQL engine of choice (Trino, Spark, DuckDB, etc.). + +## Next steps + +- Central concepts: [[Structured Data Lake with SMQ and SQL]] +- SMQ overview: [[Seaweed Message Queue]] + + diff --git a/PostgreSQL-compatible-Server-weed-db.md b/PostgreSQL-compatible-Server-weed-db.md new file mode 100644 index 0000000..0c54481 --- /dev/null +++ b/PostgreSQL-compatible-Server-weed-db.md @@ -0,0 +1,106 @@ +# PostgreSQL-compatible Server (weed db) + +SeaweedFS exposes a PostgreSQL wire protocol server so you can connect with any PostgreSQL client (psql, JDBC/ODBC, BI tools) and run SQL over Seaweed Message Queue (SMQ) topics. It is read-only and designed for analytics: query live streams and Parquet-backed history without ETL. + +## What you get + +- PostgreSQL wire protocol compatibility (psql, JDBC/ODBC, BI tools) +- Query live + historical data (hybrid scanner: brokers + Parquet files) +- Simple ops: stateless server, MD5 auth, optional TLS +- Works alongside pub/sub and Kafka ingestion — one structured data lake + +## Start the server + +```bash +# Development (no auth) +weed db + +# Production (MD5 auth) +weed db -auth=md5 -users='{"admin":"secret","analyst":"readonly"}' -host=0.0.0.0 -port=5432 + +# Credentials from file (recommended) +echo '{"admin":"secret","analyst":"readonly"}' > /etc/seaweedfs/users.json +weed db -auth=md5 -users="@/etc/seaweedfs/users.json" + +# With TLS +auth_users='@/etc/seaweedfs/users.json' +weed db -auth=md5 -users="$auth_users" -tls-cert=/etc/ssl/server.crt -tls-key=/etc/ssl/server.key +``` + +Key options: +- `-auth`: `trust` (dev), `password`, `md5` (recommended) +- `-users`: inline JSON or `@/path/to/users.json` +- `-master`: SeaweedFS master (comma-separated for HA) +- `-max-connections`, `-idle-timeout`, `-database` +- `-tls-cert`, `-tls-key` for encrypted connections + +## Connect with clients + +```bash +# psql +PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default +psql "postgresql://admin:secret@localhost:5432/default" +``` + +```java +// JDBC +String url = "jdbc:postgresql://localhost:5432/default"; +Connection conn = DriverManager.getConnection(url, "admin", "secret"); +``` + +```python +# psycopg2 +import psycopg2 +conn = psycopg2.connect(host="localhost", port=5432, user="admin", password="secret", database="default") +``` + +BI tools: use the native PostgreSQL connector (host, port 5432, database `default`, user/password). + +## SQL you can run + +Schema/navigation: +```sql +SHOW DATABASES; -- MQ namespaces +USE my_namespace; -- switch database +SHOW TABLES; -- topics in namespace +DESCRIBE topic_name; -- schema +``` + +Queries (read-only): +```sql +SELECT * FROM events LIMIT 10; +SELECT * FROM events WHERE _ts >= '2025-01-01' LIMIT 1000; +SELECT COUNT(*) FROM events; -- fast-path optimized +SELECT MIN(timestamp), MAX(timestamp) FROM events; -- fast-path + +-- System columns available on all tables +SELECT _ts, _key, _source, * FROM events LIMIT 5; +``` + +Notes: +- `_ts` supports convenient string-to-timestamp parsing in WHERE clauses +- Live + Parquet results are merged transparently + +## Limitations (current) + +- Read-only: no INSERT/UPDATE/DELETE, no DDL (CREATE/DROP/ALTER) +- Not supported yet: `JOIN`, `ORDER BY`, `GROUP BY`, `HAVING`, window functions, subqueries + +## Operations and scaling + +- Stateless server; run multiple `weed db` instances behind a load balancer +- Use MD5 auth and TLS in production +- Tune `-max-connections` and `-idle-timeout` + +## How this fits your data lake + +- Producers (Kafka or pub/sub) write schematized messages into SMQ +- SeaweedFS persists to Parquet; brokers serve live data +- PostgreSQL-compatible server exposes SQL over both — one data lake + +## Related + +- [[Structured Data Lake with SMQ and SQL]] +- [[SQL Queries on Message Queue]] +- [[SQL Quick Reference]] +- [[Seaweed Message Queue]] diff --git a/Pub-Sub-to-SMQ-to-SQL.md b/Pub-Sub-to-SMQ-to-SQL.md new file mode 100644 index 0000000..f320aa6 --- /dev/null +++ b/Pub-Sub-to-SMQ-to-SQL.md @@ -0,0 +1,67 @@ +# Pub/Sub to SMQ to SQL: From Streams to Tables + +Seaweed Message Queue (SMQ) bridges your live pub/sub streams and your analytical world. You publish structured messages, SMQ brokers stream them in real time, and at the same time SeaweedFS persists them into Parquet files so you can query with SQL engines later. + +> New here? See the bigger picture: [[Structured Data Lake with SMQ and SQL]]. + +## Why this matters + +- Real-time and batch in one pipeline: stream to subscribers now, query with SQL later. +- Store once, use twice: messages land in Parquet (columnar, compressed), great for analytics. +- Stateless brokers, disaggregated storage: scale brokers independently of storage. + +## Architecture at a glance + +``` +Publishers => SMQ Agent (gRPC) => SMQ Brokers => Subscribers + \ + +--> SeaweedFS (Parquet) => SQL Engines +``` + +- **Publish**: Send structured messages with a schema. +- **Stream**: Subscribers process messages with consumer groups and offsets. +- **Persist**: Messages are compacted/organized into Parquet files in SeaweedFS. +- **Query**: Point your SQL engines to the Parquet location. + +## What you publish + +Messages are structured records defined by a schema. SMQ validates these on publish and keeps ordering guarantees per key while enabling high concurrency via a sliding window. + +## What you query + +Parquet files written by SMQ are queryable by your favorite SQL engines: + +- Trino/Presto +- Spark SQL +- DuckDB +- ClickHouse (via file table engines) + +Point them to the Parquet path in SeaweedFS and query away. + +## Quick start + +1) Start a broker and an agent: + +```bash +weed mq.broker -port=17777 -master=localhost:9333 +weed mq.agent -port=16777 -broker=localhost:17777 +``` + +2) Define a schema and publish: + +```go +type MyEvent struct { + Key []byte + UserId int64 + Action string +} +``` + +3) Subscribe in real time. Your consumers get the stream; your data team gets Parquet. + +## Where to go next + +- Central concepts: [[Structured Data Lake with SMQ and SQL]] +- Messaging basics: [[Seaweed Message Queue]] + + diff --git a/Structured-Data-Lake-with-SMQ-and-SQL.md b/Structured-Data-Lake-with-SMQ-and-SQL.md new file mode 100644 index 0000000..1d90948 --- /dev/null +++ b/Structured-Data-Lake-with-SMQ-and-SQL.md @@ -0,0 +1,52 @@ +# Structured Data Lake with SMQ and SQL + +SeaweedFS + Seaweed Message Queue (SMQ) gives you a unified pipeline: produce structured messages, process them in real time, and query the same data with SQL. Whether producers speak Kafka or a simple pub/sub gRPC API, they both write schematized messages into the same data lake. + +## Core ideas + +- Schematized messages can be queried directly with SQL (no ETL required) +- SMQ brokers are computation-only nodes and scale linearly with demand +- Structured data is written as messages and can be queried in real time +- Together, SeaweedFS + SMQ form a data lake for structured data (hot streams + Parquet) + +## Ingestion paths + +Two equivalent ways to ingest structured messages: + +- Kafka clients → [[Kafka to Kafka Gateway to SMQ to SQL]] +- Pub/Sub clients → [[Pub-Sub to SMQ to SQL]] + +Both end up with the same outcomes: live streams for subscribers and Parquet files in SeaweedFS for SQL engines. + +## Architecture + +``` +Producers (Kafka or Pub/Sub) ==> SMQ Brokers ==> Subscribers (real-time) + \ + +--> SeaweedFS (Parquet) ==> SQL Engines +``` + +## Querying the lake + +Point your SQL engines at the Parquet paths: + +- Trino/Presto +- Spark SQL +- DuckDB +- ClickHouse (file table engines) + +Examples are available in the ingestion pages. + +## Operate at scale + +- Scale SMQ brokers horizontally; they are stateless computation nodes +- Storage is disaggregated (SeaweedFS) for durable, efficient Parquet files + +## Learn more + +- Messaging basics: [[Seaweed Message Queue]] +- PostgreSQL-compatible server: [[PostgreSQL-compatible Server weed db]] +- Kafka ingestion: [[Kafka to Kafka Gateway to SMQ to SQL]] +- Pub/Sub ingestion: [[Pub-Sub to SMQ to SQL]] + + diff --git a/_Sidebar.md b/_Sidebar.md index 9d0656e..66b885a 100644 --- a/_Sidebar.md +++ b/_Sidebar.md @@ -115,9 +115,13 @@ * [[Filer Metadata Events]] ### Messaging +* [[Structured Data Lake with SMQ and SQL]] * [[Seaweed Message Queue]] * [[SQL Queries on Message Queue]] * [[SQL Quick Reference]] +* [[PostgreSQL-compatible Server weed db]] +* [[Pub-Sub to SMQ to SQL]] +* [[Kafka to Kafka Gateway to SMQ to SQL]] ### Use Cases * [[Use Cases]]