mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-22 12:17:04 +01:00
Compare commits
60
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d84ecca2f | ||
|
|
a1d2ebbb9b | ||
|
|
371d320854 | ||
|
|
84cc179b80 | ||
|
|
c8002d63bb | ||
|
|
9b642d4636 | ||
|
|
a882565cd7 | ||
|
|
de2fc03c7d | ||
|
|
21add33b6e | ||
|
|
9a03c09396 | ||
|
|
f40267db44 | ||
|
|
6ee7c4c5c7 | ||
|
|
9c427b265f | ||
|
|
97f8ca13ff | ||
|
|
780760fccc | ||
|
|
a3e97ec709 | ||
|
|
23d8c62844 | ||
|
|
4bdbfbaaa8 | ||
|
|
e73c23a4ee | ||
|
|
8c3d8fcea0 | ||
|
|
bb1ba8b247 | ||
|
|
593cfee85b | ||
|
|
03065155db | ||
|
|
f8cfea0dcf | ||
|
|
c90da999f2 | ||
|
|
379abbe278 | ||
|
|
c601712cc4 | ||
|
|
ab738bfd33 | ||
|
|
222e96d6ab | ||
|
|
a724f4296c | ||
|
|
f05d743459 | ||
|
|
03a75cbc52 | ||
|
|
efa9d304d2 | ||
|
|
b7f5d04428 | ||
|
|
ece2f7a7c2 | ||
|
|
610346ae8c | ||
|
|
df89afbef4 | ||
|
|
1253b97303 | ||
|
|
a1d8739464 | ||
|
|
1cfd9bbb4c | ||
|
|
0de6347b11 | ||
|
|
6936bbdc3e | ||
|
|
4d06742664 | ||
|
|
2874bd05fb | ||
|
|
1144bf15d4 | ||
|
|
8d2b417c69 | ||
|
|
7ed26deaae | ||
|
|
da0d423bd9 | ||
|
|
2095c905ba | ||
|
|
6b6a268903 | ||
|
|
79daffb2d2 | ||
|
|
207d06863a | ||
|
|
7e348951c2 | ||
|
|
8b09c96cec | ||
|
|
0fb6c0e77d | ||
|
|
60b06b6881 | ||
|
|
dbcf6850b6 | ||
|
|
04eff7d613 | ||
|
|
4feb30deff | ||
|
|
117ea58a83 |
No files matched your search
@@ -86,7 +86,11 @@ jobs:
|
||||
set -e
|
||||
dd if=/dev/urandom of=generated bs=1M count=32
|
||||
aws --no-verify-ssl s3 cp --no-progress generated s3://bucket/test-multipart
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded
|
||||
for i in 1 2 3 4 5; do
|
||||
aws --no-verify-ssl s3 cp --no-progress s3://bucket/test-multipart downloaded && break
|
||||
if [ "$i" -eq 5 ]; then exit 1; fi
|
||||
sleep 2
|
||||
done
|
||||
diff -q generated downloaded
|
||||
rm -f generated downloaded
|
||||
|
||||
|
||||
@@ -225,7 +225,7 @@ The full operation list is in [Amazon S3 API][AmazonS3API], and [Supported APIs
|
||||
|
||||
SeaweedFS is a lakehouse in one system. [S3 Table Buckets][S3TableBucket] hold Apache Iceberg tables by default, or [Lance][LanceCatalog] tables for vectors and multimodal data, and the built-in [Iceberg REST Catalog][IcebergCatalog] and Lance namespace serve them directly. There is no Hive Metastore, Glue, or separate catalog service to deploy, secure, and back up.
|
||||
|
||||
* Query engines operate on the same tables at the same time: [Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], [Apache Doris][DorisIceberg], [RisingWave][RisingWaveIceberg], ClickHouse, and [LanceDB][LanceDB]. Catalog commits are atomic compare-and-swap, so concurrent writers are safe. [Lakekeeper][Lakekeeper] can front the same storage with STS-vended credentials.
|
||||
* Query engines operate on the same tables at the same time: [Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], [Apache Doris][DorisIceberg], [RisingWave][RisingWaveIceberg], ClickHouse, and [LanceDB][LanceDB]. Catalog commits are atomic compare-and-swap, so concurrent writers are safe. [Lakekeeper][Lakekeeper] can front the same storage with STS-vended credentials. [OLake Go][OLake] is a high-performance, open-source EL (Extract–Load) platform that bridges operational databases and open lakehouse storage - writing Apache Iceberg tables or Parquet files on Amazon S3.
|
||||
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan file removal, and manifest rewriting, configured per bucket or table through the S3 Tables maintenance APIs, and the same for [Lance][LanceMaintenance].
|
||||
* IAM at the bucket, namespace, and table level with standard bucket policies, see [S3 Tables Security][S3TablesSecurity].
|
||||
* A [Hadoop compatible file system][Hadoop] for Spark, Flink, and HBase.
|
||||
@@ -487,6 +487,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
|
||||
[LanceDB]: https://github.com/seaweedfs/seaweedfs/wiki/LanceDB-Integration
|
||||
[Lakekeeper]: https://github.com/seaweedfs/seaweedfs/wiki/Lakekeeper-Iceberg-Integration
|
||||
[OLake]: https://github.com/seaweedfs/seaweedfs/wiki/OLake-Go-Iceberg-Integration
|
||||
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
|
||||
[LanceMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Lance-Maintenance-Worker
|
||||
[S3TablesSecurity]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Tables-Security
|
||||
|
||||
+1099
-1067
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 55 KiB |
@@ -68,7 +68,10 @@ seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
|
||||
| `--pprof` | `false` | Enable pprof HTTP handlers |
|
||||
|
||||
Set `RUST_LOG=debug` (or `trace`, `info`, `warn`) for log level control.
|
||||
Set `SEAWEED_WRITE_QUEUE=1` to enable batched async write processing.
|
||||
|
||||
Durable (`fsync=true`) uploads always go through per-volume batching workers
|
||||
that share one `.dat`/`.idx` sync per batch, matching the Go volume server;
|
||||
non-fsync writes take the synchronous path.
|
||||
|
||||
## Features
|
||||
|
||||
|
||||
@@ -59,6 +59,8 @@ service Seaweed {
|
||||
}
|
||||
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
|
||||
}
|
||||
rpc ResumeVolumeGrowth (ResumeVolumeGrowthRequest) returns (ResumeVolumeGrowthResponse) {
|
||||
}
|
||||
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
|
||||
}
|
||||
rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
|
||||
@@ -119,6 +121,12 @@ message Heartbeat {
|
||||
// place of `volumes`. A master that does not understand this never sets
|
||||
// volume_digest_supported, so it keeps being sent the whole list.
|
||||
repeated VolumeInformationMessage changed_volumes = 28;
|
||||
|
||||
// Highest volume id this server has ever observed, persisted locally. Fed
|
||||
// by HeartbeatResponse.max_volume_id echoes and by AllocateVolume requests.
|
||||
uint32 max_volume_id_seen = 29;
|
||||
// Topology id of the master cluster this server last heartbeated to.
|
||||
string last_seen_topology_id = 30;
|
||||
}
|
||||
|
||||
message HeartbeatResponse {
|
||||
@@ -135,6 +143,12 @@ message HeartbeatResponse {
|
||||
// The master compares volume digests, so a server that reports one may send
|
||||
// changed_volumes in place of its whole list.
|
||||
bool volume_digest_supported = 9;
|
||||
// The leader's current allocation bound; servers persist it and report it
|
||||
// back as max_volume_id_seen so the bound survives master state loss.
|
||||
uint32 max_volume_id = 10;
|
||||
// The leader's cluster identity; a different value reported back later
|
||||
// means the master's state was rebuilt.
|
||||
string topology_id = 11;
|
||||
}
|
||||
|
||||
message VolumeInformationMessage {
|
||||
@@ -305,6 +319,18 @@ message VolumeGrowRequest {
|
||||
string data_node = 7;
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
string disk_type = 9;
|
||||
// If > 0, grow only when the average fill ratio of the layout's writable
|
||||
// volumes has reached this threshold (e.g. 0.6). Lets callers pre-grow on
|
||||
// fill level with one call, instead of listing volumes to compute the
|
||||
// ratio themselves. Ignored when there are no writable volumes.
|
||||
float grow_avg_fill_threshold = 10;
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthRequest {
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthResponse {
|
||||
bool was_frozen = 1;
|
||||
}
|
||||
|
||||
message AssignResponse {
|
||||
@@ -611,4 +637,9 @@ message RaftLeadershipTransferResponse {
|
||||
}
|
||||
|
||||
message VolumeGrowResponse {
|
||||
// Average fill ratio of the layout's writable volumes at request time
|
||||
// (0 when there are none). Reported whether or not growth was triggered.
|
||||
float avg_fill = 1;
|
||||
// Number of writable volumes at request time.
|
||||
uint32 writable_count = 2;
|
||||
}
|
||||
@@ -303,8 +303,6 @@ pub struct VolumeServerConfig {
|
||||
pub grpc_allowed_wildcard_domain: String,
|
||||
pub grpc_volume_allowed_common_names: Vec<String>,
|
||||
pub tls_policy: TlsPolicy,
|
||||
/// Enable batched write queue for improved throughput under load.
|
||||
pub enable_write_queue: bool,
|
||||
/// Path to security.toml — stored for SIGHUP reload.
|
||||
pub security_file: String,
|
||||
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation
|
||||
@@ -968,8 +966,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
|
||||
grpc_allowed_wildcard_domain: sec.grpc_allowed_wildcard_domain,
|
||||
grpc_volume_allowed_common_names: sec.grpc_volume_allowed_common_names,
|
||||
tls_policy: sec.tls_policy,
|
||||
enable_write_queue: env_string(env, "SEAWEED_WRITE_QUEUE")
|
||||
.is_some_and(|v| v == "1" || v == "true"),
|
||||
security_file: cli.security_file,
|
||||
allow_untrusted_remote_endpoints: cli.allow_untrusted_remote_endpoints,
|
||||
}
|
||||
|
||||
@@ -433,12 +433,11 @@ async fn run(
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize the batched write queue if enabled
|
||||
if config.enable_write_queue {
|
||||
info!("Batched write queue enabled");
|
||||
let wq = WriteQueue::new(state.clone(), 128);
|
||||
let _ = state.write_queue.set(wq);
|
||||
}
|
||||
// The write queue is always on: durable (fsync) uploads go through it so
|
||||
// a batch shares one .dat/.idx sync, the way Go's per-volume async writer
|
||||
// group-commits. Non-fsync writes stay on the synchronous path.
|
||||
let wq = WriteQueue::new(state.clone(), 128);
|
||||
let _ = state.write_queue.set(wq);
|
||||
|
||||
// Set initial metric gauges for concurrent limits and max volumes
|
||||
metrics::CONCURRENT_UPLOAD_LIMIT.set(state.concurrent_upload_limit);
|
||||
|
||||
@@ -431,7 +431,6 @@ mod tests {
|
||||
grpc_allowed_wildcard_domain: String::new(),
|
||||
grpc_volume_allowed_common_names: vec![],
|
||||
tls_policy: TlsPolicy::default(),
|
||||
enable_write_queue: false,
|
||||
security_file: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::pb::filer_pb;
|
||||
use crate::pb::master_pb;
|
||||
use crate::pb::volume_server_pb;
|
||||
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
|
||||
use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from};
|
||||
use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from};
|
||||
use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder;
|
||||
use crate::storage::needle::needle::{self, Needle};
|
||||
use crate::storage::types::*;
|
||||
@@ -53,6 +53,12 @@ fn needle_not_found(needle_id: NeedleId) -> Status {
|
||||
Status::not_found(format!("needle not found {}", needle_id.0))
|
||||
}
|
||||
|
||||
/// Mirrors Go's `isNotLeaderErr`: the master returns raft's NotLeaderError,
|
||||
/// which arrives as an Unknown-status message.
|
||||
fn is_not_leader_status(e: &Status) -> bool {
|
||||
e.message().contains("Not current leader")
|
||||
}
|
||||
|
||||
/// Deadline from the caller's `grpc-timeout` header (TimeoutValue +
|
||||
/// H|M|S|m|u|n unit, per the gRPC-over-HTTP2 spec). Tonic's GrpcTimeout layer
|
||||
/// drops an over-deadline handler future anyway; parsing it here lets a queued
|
||||
@@ -344,22 +350,46 @@ impl VolumeGrpcService {
|
||||
is_readonly: bool,
|
||||
) -> Result<(), Status> {
|
||||
// VolumeMarkReadonly mutates raft-replicated master topology, so it must
|
||||
// reach the leader. Prefer the live leader the heartbeat is talking to
|
||||
// (current_master_url), falling back to the static seed before the first
|
||||
// heartbeat — mirrors store_ec.rs and Go's vs.GetMaster(). Sending it to
|
||||
// the static seed fails "not current leader" after any master failover.
|
||||
let master_url = {
|
||||
let live = self.state.current_master_url.read().await.clone();
|
||||
if !live.is_empty() {
|
||||
live
|
||||
} else {
|
||||
self.state.master_url.clone()
|
||||
}
|
||||
};
|
||||
// reach the leader the heartbeat confirmed. On "Not current leader",
|
||||
// ask the seed masters who leads and retry there — mirrors Go's
|
||||
// notifyMasterVolumeReadonly.
|
||||
let master_url = self.state.current_master().await;
|
||||
if master_url.is_empty() {
|
||||
return Ok(());
|
||||
if !self.state.has_master {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Status::unavailable("no master leader known"));
|
||||
}
|
||||
let grpc_addr = parse_grpc_address(&master_url).map_err(|e| {
|
||||
match self
|
||||
.volume_mark_readonly_on_master(&master_url, info, is_readonly)
|
||||
.await
|
||||
{
|
||||
Err(e) if is_not_leader_status(&e) => {
|
||||
let leader = super::heartbeat::lookup_leader_from_peers(
|
||||
&self.state.master_urls,
|
||||
self.state.outgoing_grpc_tls.as_ref(),
|
||||
)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
Status::unavailable(format!(
|
||||
"master {} rejected mark-readonly and no peer named a leader",
|
||||
master_url
|
||||
))
|
||||
})?;
|
||||
self.volume_mark_readonly_on_master(&leader, info, is_readonly)
|
||||
.await
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn volume_mark_readonly_on_master(
|
||||
&self,
|
||||
master_url: &str,
|
||||
info: &MasterVolumeInfo,
|
||||
is_readonly: bool,
|
||||
) -> Result<(), Status> {
|
||||
let grpc_addr = parse_grpc_address(master_url).map_err(|e| {
|
||||
Status::internal(format!("invalid master address {}: {}", master_url, e))
|
||||
})?;
|
||||
let channel = connect_channel(
|
||||
@@ -1353,6 +1383,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Go's CommitCompact waits out leased readers under
|
||||
// dataFileAccessLock — do it without holding the store
|
||||
// guard, so a slow download stalls only this vacuum.
|
||||
let _sync_guard = vol.data_file_sync_lock().lock().unwrap();
|
||||
let _lease = vol.instance().write_lock();
|
||||
let is_read_only = vol.is_read_only();
|
||||
vol.commit_compact_held()
|
||||
@@ -2317,9 +2348,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// owns the .ecx on disk (volume not yet mounted — relevant
|
||||
// when shards stream in mid-rebuild before any
|
||||
// VolumeEcShardsMount has happened; see #9212), then any
|
||||
// HDD, then any disk. Pass the build's default data-shard
|
||||
// count for free-slot maths; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
// HDD, then any disk. The free-slot maths uses the
|
||||
// volume's own data-shard count from its .vif so a
|
||||
// custom-ratio volume is not scored as 10+4.
|
||||
let vid = VolumeId(info.volume_id);
|
||||
let dir = if info.disk_id > 0 {
|
||||
let count = store.locations.len();
|
||||
@@ -2334,11 +2365,13 @@ impl VolumeServer for VolumeGrpcService {
|
||||
} else {
|
||||
// The mounted-volume refusal above means no disk
|
||||
// holds an in-memory claim here.
|
||||
let data_shards =
|
||||
store.lookup_ec_data_shards(&info.collection, vid);
|
||||
store
|
||||
.find_ec_shard_target_location(
|
||||
&info.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
data_shards,
|
||||
&[],
|
||||
)
|
||||
.map(|i| store.locations[i].directory.clone())
|
||||
@@ -3288,9 +3321,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// volume mounted, then a disk that owns the .ecx on disk (volume
|
||||
// not yet mounted — relevant for ec.rebuild, where only the
|
||||
// first shard carries .ecx and subsequent shards must land on
|
||||
// the same disk; see #9212), then any HDD, then any disk. Pass
|
||||
// the build's default data-shard count; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
// the same disk; see #9212), then any HDD, then any disk. The
|
||||
// free-slot maths uses the volume's own data-shard count from its
|
||||
// .vif so a custom-ratio volume is not scored as 10+4.
|
||||
let (dest_dir, dest_idx_dir) = {
|
||||
let store = self.state.store.read().unwrap();
|
||||
let count = store.locations.len();
|
||||
@@ -3322,10 +3355,11 @@ impl VolumeServer for VolumeGrpcService {
|
||||
req.volume_id, shard_ids, dirs
|
||||
)));
|
||||
}
|
||||
let data_shards = store.lookup_ec_data_shards(&req.collection, vid);
|
||||
match store.find_ec_shard_target_location(
|
||||
&req.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
data_shards,
|
||||
&shard_ids,
|
||||
) {
|
||||
Some(i) => {
|
||||
@@ -4723,6 +4757,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// backend swap: between clearing the remote-only write
|
||||
// restriction and attaching the local .dat a write would
|
||||
// pass check_writable and then fail on a missing dat_file.
|
||||
let _sync_guard = vol.data_file_sync_lock().lock().unwrap();
|
||||
let _lease = vol.instance().write_lock();
|
||||
|
||||
// The store guard dropped before the lease was taken, so an
|
||||
@@ -5771,7 +5806,8 @@ impl VolumeCopyJob {
|
||||
(&self.dest.data_base_name, self.dest.has_remote_dat);
|
||||
let mut preallocate_size: i64 = 0;
|
||||
if !has_remote_dat {
|
||||
let grpc_addr = super::heartbeat::to_grpc_address(&state.master_url);
|
||||
let master = state.current_master().await;
|
||||
let grpc_addr = super::heartbeat::to_grpc_address(&master);
|
||||
// Race the master-configuration RPC against the caller's
|
||||
// response channel: a stalled master (or a slow leader
|
||||
// election) would otherwise hold the task and its .note
|
||||
@@ -5796,7 +5832,7 @@ impl VolumeCopyJob {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("get master {} configuration: {}", state.master_url, e);
|
||||
tracing::warn!("get master {} configuration: {}", master, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -603,7 +603,7 @@ async fn batch_delete_file_ids(
|
||||
let locations = lookup_volume(
|
||||
&state.http_client,
|
||||
&state.outgoing_http_scheme,
|
||||
&state.master_url,
|
||||
&state.current_master().await,
|
||||
volume_id,
|
||||
)
|
||||
.await
|
||||
@@ -675,7 +675,7 @@ async fn do_replicated_request(
|
||||
let locations = lookup_volume(
|
||||
&state.http_client,
|
||||
&state.outgoing_http_scheme,
|
||||
&state.master_url,
|
||||
&state.current_master().await,
|
||||
vid,
|
||||
)
|
||||
.await
|
||||
@@ -832,7 +832,7 @@ async fn proxy_or_redirect_to_target(
|
||||
let locations = match lookup_volume(
|
||||
&state.http_client,
|
||||
&state.outgoing_http_scheme,
|
||||
&state.master_url,
|
||||
&state.current_master().await,
|
||||
vid.0,
|
||||
)
|
||||
.await
|
||||
@@ -1238,7 +1238,7 @@ async fn proxy_missing_volume(
|
||||
let query_string = uri.query().unwrap_or("").to_string();
|
||||
if is_proxied_query(&query_string)
|
||||
|| state.read_mode == ReadMode::Local
|
||||
|| state.master_url.is_empty()
|
||||
|| state.current_master().await.is_empty()
|
||||
{
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
@@ -1282,7 +1282,7 @@ async fn check_download_limit(
|
||||
// Go tries proxy to replica once before the wait loop
|
||||
// (checkDownloadLimit); it does not retry on each wakeup.
|
||||
let should_try_replica =
|
||||
!is_proxied_query(&query_string) && !state.master_url.is_empty() && {
|
||||
!is_proxied_query(&query_string) && !state.current_master().await.is_empty() && {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
@@ -2433,6 +2433,9 @@ fn is_zero_u32(v: &u32) -> bool {
|
||||
*v == 0
|
||||
}
|
||||
|
||||
/// Store an uploaded needle on the local volume — durable (fsync) uploads
|
||||
/// through the per-volume write queue, the rest synchronously — and then
|
||||
/// replicate it if the volume's placement calls for more copies.
|
||||
pub async fn post_handler(
|
||||
State(state): State<Arc<VolumeServerState>>,
|
||||
request: Request<Body>,
|
||||
@@ -2953,19 +2956,13 @@ pub async fn post_handler(
|
||||
let needle_etag = n.etag();
|
||||
let needle_has_name = n.has_name();
|
||||
|
||||
let write_result = if let Some(wq) = state.write_queue.get() {
|
||||
wq.submit(vid, n, fsync).await
|
||||
} else {
|
||||
let vol = {
|
||||
let store = state.store.read().unwrap();
|
||||
store.writable_volume(vid)
|
||||
};
|
||||
vol.and_then(|vol| vol.write_needle(&mut n, true, fsync))
|
||||
};
|
||||
let write_result = write_upload_needle(&state, vid, n, fsync).await;
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
if !is_replicate && write_result.is_ok() && !state.master_url.is_empty() {
|
||||
// No leader-known gate: when the lookup cannot run, the write must fail
|
||||
// rather than be acknowledged on fewer copies than required.
|
||||
if !is_replicate && write_result.is_ok() {
|
||||
let needs_replication = {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
@@ -3055,6 +3052,28 @@ pub async fn post_handler(
|
||||
resp
|
||||
}
|
||||
|
||||
/// Append an uploaded needle the way Go's `writeNeedle2` does: a durable
|
||||
/// write goes to the volume's batching worker, which group-commits the
|
||||
/// batch's .dat/.idx syncs, while anything else — a non-durable write, or a
|
||||
/// durable one landing while the server is stopping and the workers are
|
||||
/// winding down — takes the synchronous path and fsyncs inline.
|
||||
async fn write_upload_needle(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
mut n: Needle,
|
||||
fsync: bool,
|
||||
) -> Result<(u64, Size, bool), crate::storage::volume::VolumeError> {
|
||||
let queued = fsync && !*state.is_stopping.read().unwrap();
|
||||
if queued && let Some(wq) = state.write_queue.get() {
|
||||
return wq.submit(vid, n, fsync).await;
|
||||
}
|
||||
let vol = {
|
||||
let store = state.store.read().unwrap();
|
||||
store.writable_volume(vid)
|
||||
};
|
||||
vol.and_then(|vol| vol.write_needle(&mut n, true, fsync))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Delete Handler
|
||||
// ============================================================================
|
||||
@@ -3341,7 +3360,7 @@ pub async fn delete_handler(
|
||||
};
|
||||
|
||||
let is_replicate = del_query.split('&').any(|p| p == "type=replicate");
|
||||
if !is_replicate && delete_result.is_ok() && !state.master_url.is_empty() {
|
||||
if !is_replicate && delete_result.is_ok() {
|
||||
let needs_replication = {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
@@ -4215,7 +4234,7 @@ async fn read_remote_chunk_needle(
|
||||
let locations = lookup_volume(
|
||||
&state.http_client,
|
||||
&state.outgoing_http_scheme,
|
||||
&state.master_url,
|
||||
&state.current_master().await,
|
||||
vid.0,
|
||||
)
|
||||
.await?;
|
||||
@@ -6654,7 +6673,8 @@ mod tests {
|
||||
let s = Arc::get_mut(&mut state).unwrap();
|
||||
// Empty self_url makes `url.contains(self_url)` true for every peer.
|
||||
s.self_url = "198.51.100.8:8080".to_string();
|
||||
s.master_url = master_url;
|
||||
s.master_url = master_url.clone();
|
||||
s.current_master_url = tokio::sync::RwLock::new(master_url);
|
||||
state
|
||||
}
|
||||
|
||||
@@ -7188,4 +7208,52 @@ mod tests {
|
||||
"the vacuum moves the needle once the stream drains"
|
||||
);
|
||||
}
|
||||
|
||||
/// Issue #801: an fsync upload takes the per-volume worker so its sync is
|
||||
/// shared across the batch (Go's `writeNeedle2` split); a non-fsync write,
|
||||
/// or an fsync write landing while the server is stopping, takes the
|
||||
/// synchronous path instead.
|
||||
#[tokio::test]
|
||||
async fn test_write_upload_needle_routes_fsync_to_queue() {
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let state = volume_test_state(&tmp);
|
||||
assert!(
|
||||
state
|
||||
.write_queue
|
||||
.set(crate::server::write_queue::WriteQueue::new(
|
||||
state.clone(),
|
||||
128,
|
||||
))
|
||||
.is_ok()
|
||||
);
|
||||
let queue = state.write_queue.get().unwrap();
|
||||
|
||||
let needle = |id: u64| {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(TEST_COOKIE),
|
||||
data: vec![id as u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
n.checksum = CRC::new(&n.data);
|
||||
n
|
||||
};
|
||||
|
||||
write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b01), true)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(queue.submits_for_test(), 1);
|
||||
|
||||
write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b02), false)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(queue.submits_for_test(), 1);
|
||||
|
||||
*state.is_stopping.write().unwrap() = true;
|
||||
write_upload_needle(&state, VolumeId(1), needle(0x6e7a_0b03), true)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(queue.submits_for_test(), 1);
|
||||
}
|
||||
}
|
||||
@@ -155,7 +155,11 @@ pub async fn run_heartbeat_with_state(
|
||||
}
|
||||
PostAction::SleepPulse => {
|
||||
new_leader = None;
|
||||
tokio::time::sleep(pulse).await;
|
||||
// The lookup spends part of the retry delay rather than
|
||||
// adding to it.
|
||||
let retry_at = tokio::time::Instant::now() + pulse;
|
||||
refresh_leader_from_peers(&config, &state, pulse).await;
|
||||
tokio::time::sleep_until(retry_at).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,10 +258,56 @@ pub async fn try_get_master_configuration(
|
||||
Ok(resp.into_inner())
|
||||
}
|
||||
|
||||
/// Publish the leader the configured masters report, so lookups stop going
|
||||
/// to the master the heartbeat just lost while the loop retries. Without an
|
||||
/// answer inside `timeout` the published master is cleared: the lost master
|
||||
/// is dead or no longer leader, and a follower answers lookups from its own
|
||||
/// cache rather than refusing them. Mirrors Go's refreshLeaderFromPeers.
|
||||
async fn refresh_leader_from_peers(
|
||||
config: &HeartbeatConfig,
|
||||
state: &Arc<VolumeServerState>,
|
||||
timeout: Duration,
|
||||
) {
|
||||
let lookup = lookup_leader_from_peers(
|
||||
&config.master_addresses,
|
||||
state.outgoing_grpc_tls.as_ref(),
|
||||
);
|
||||
match tokio::time::timeout(timeout, lookup).await {
|
||||
Ok(Some(leader)) => publish_current_master(state, &leader).await,
|
||||
_ => state.current_master_url.write().await.clear(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask each seed master for the raft leader and return the first answer, or
|
||||
/// None when none replies with one. Mirrors Go's
|
||||
/// `operation.LookupRaftLeaderMaster`.
|
||||
pub async fn lookup_leader_from_peers(
|
||||
master_addrs: &[String],
|
||||
tls: Option<&super::grpc_client::OutgoingGrpcTlsConfig>,
|
||||
) -> Option<String> {
|
||||
for master_addr in master_addrs {
|
||||
let grpc_addr = to_grpc_address(master_addr);
|
||||
if let Ok(resp) = try_get_master_configuration(&grpc_addr, tls).await
|
||||
&& !resp.leader.is_empty()
|
||||
{
|
||||
return Some(resp.leader);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn is_stopping(state: &VolumeServerState) -> bool {
|
||||
*state.is_stopping.read().unwrap()
|
||||
}
|
||||
|
||||
/// Store the master lookups should go to, in the address form it was
|
||||
/// announced in — `host:port` or `host:port.grpcPort` — so an explicit gRPC
|
||||
/// port survives to the gRPC callers, mirroring Go's setCurrentMaster.
|
||||
/// The heartbeat calls this only for a master known to lead.
|
||||
async fn publish_current_master(state: &Arc<VolumeServerState>, master: &str) {
|
||||
*state.current_master_url.write().await = master.to_string();
|
||||
}
|
||||
|
||||
fn duplicate_uuid_retry_delay(retry_count: u32) -> Duration {
|
||||
Duration::from_secs((1u64 << retry_count) * 2)
|
||||
}
|
||||
@@ -430,14 +480,6 @@ async fn do_heartbeat(
|
||||
let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
|
||||
|
||||
info!("Heartbeat stream established with {}", grpc_addr);
|
||||
// Publish the master we're now talking to in canonical http host:port
|
||||
// form so Ping admission can recognise it once a leader change moves us
|
||||
// off the seed list. Mirrors Go's vs.setCurrentMaster(masterAddress).
|
||||
{
|
||||
let normalised = super::volume_server::to_http_address(current_master).into_owned();
|
||||
let mut guard = state.current_master_url.write().await;
|
||||
*guard = normalised;
|
||||
}
|
||||
if is_stopping(state) {
|
||||
state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
send_deregister_heartbeat(config, state, &tx).await;
|
||||
@@ -451,13 +493,31 @@ async fn do_heartbeat(
|
||||
volume_tick.tick().await;
|
||||
ec_tick.tick().await;
|
||||
|
||||
let mut confirmed_leader = false;
|
||||
loop {
|
||||
tokio::select! {
|
||||
resp = response_stream.message() => {
|
||||
match resp {
|
||||
Ok(Some(hb_resp)) => {
|
||||
// Match Go ordering: DuplicatedUuids first, then volume
|
||||
// options, then leader redirect.
|
||||
// A reply names the leader or comes from it, so it
|
||||
// settles where lookups go before anything else on the
|
||||
// reply is handled: a follower reply carries no other
|
||||
// fields, and a leader stays the lookup target even
|
||||
// when it rejects this registration. Mirrors Go.
|
||||
if !hb_resp.leader.is_empty() {
|
||||
if super::volume_server::to_http_address(current_master)
|
||||
!= super::volume_server::to_http_address(&hb_resp.leader)
|
||||
{
|
||||
publish_current_master(state, &hb_resp.leader).await;
|
||||
return Ok(Some(hb_resp.leader));
|
||||
}
|
||||
} else if !confirmed_leader {
|
||||
// The leader answers every new stream without
|
||||
// naming a leader, so this peer is the one lookups
|
||||
// go to.
|
||||
confirmed_leader = true;
|
||||
publish_current_master(state, current_master).await;
|
||||
}
|
||||
if !hb_resp.duplicated_uuids.is_empty() {
|
||||
let uuids = hb_resp.duplicated_uuids.clone();
|
||||
let duplicate_dirs = off_runtime(config, state, move |_, state| {
|
||||
@@ -474,6 +534,14 @@ async fn do_heartbeat(
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let echo_max_volume_id = hb_resp.max_volume_id;
|
||||
let echo_topology_id = hb_resp.topology_id.clone();
|
||||
off_runtime(config, state, move |_, state| {
|
||||
let store = state.store.read().unwrap();
|
||||
store.update_max_volume_id_seen(echo_max_volume_id);
|
||||
store.update_last_seen_topology_id(&echo_topology_id);
|
||||
})
|
||||
.await?;
|
||||
let options_changed =
|
||||
volume_options_pass(config, state, &hb_resp).await?;
|
||||
if options_changed && maybe_adjust_volume_max(config, state).await? {
|
||||
@@ -494,11 +562,6 @@ async fn do_heartbeat(
|
||||
if metrics_changed {
|
||||
state.metrics_notify.notify_waiters();
|
||||
}
|
||||
// Match Go: only redirect if leader is non-empty AND
|
||||
// different from the current master we're connected to.
|
||||
if !hb_resp.leader.is_empty() && current_master != hb_resp.leader {
|
||||
return Ok(Some(hb_resp.leader));
|
||||
}
|
||||
}
|
||||
Ok(None) => return Ok(None),
|
||||
Err(e) => return Err(Box::new(e)),
|
||||
@@ -1360,6 +1423,8 @@ fn build_heartbeat_with_ec_status(
|
||||
grpc_port: config.grpc_port as u32,
|
||||
location_uuids,
|
||||
disk_tags,
|
||||
max_volume_id_seen: store.max_volume_id_seen(),
|
||||
last_seen_topology_id: store.last_seen_topology_id(),
|
||||
..Default::default()
|
||||
};
|
||||
(heartbeat, volumes, actions)
|
||||
|
||||
@@ -170,7 +170,6 @@ mod tests {
|
||||
grpc_allowed_wildcard_domain: String::new(),
|
||||
grpc_volume_allowed_common_names: vec![],
|
||||
tls_policy: TlsPolicy::default(),
|
||||
enable_write_queue: false,
|
||||
security_file: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1139,16 +1139,9 @@ async fn cached_lookup_ec_shard_locations(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<HashMap<ShardId, Vec<String>>> {
|
||||
let master = {
|
||||
let live = state.current_master_url.read().await.clone();
|
||||
if !live.is_empty() {
|
||||
live
|
||||
} else {
|
||||
state.master_url.clone()
|
||||
}
|
||||
};
|
||||
let master = state.current_master().await;
|
||||
if master.is_empty() {
|
||||
return Err(io::Error::other("no master configured for ec shard lookup"));
|
||||
return Err(io::Error::other("no master leader known for ec shard lookup"));
|
||||
}
|
||||
|
||||
let grpc_addr =
|
||||
|
||||
@@ -77,14 +77,17 @@ pub struct VolumeServerState {
|
||||
pub pre_stop_seconds: u32,
|
||||
/// Notify heartbeat to send an immediate update when volume state changes.
|
||||
pub volume_state_notify: tokio::sync::Notify,
|
||||
/// Optional batched write queue for improved throughput under load.
|
||||
/// Per-volume write workers that group-commit durable (fsync) uploads.
|
||||
/// Set once at startup in production; a test state may leave it unset,
|
||||
/// which sends every write down the synchronous path.
|
||||
pub write_queue: std::sync::OnceLock<WriteQueue>,
|
||||
/// Read mode: local, proxy, or redirect for non-local volumes.
|
||||
pub read_mode: ReadMode,
|
||||
/// If true, FetchAndWriteNeedle skips remote S3 endpoint validation,
|
||||
/// allowing arbitrary (incl. loopback / link-local / metadata) hosts.
|
||||
pub allow_untrusted_remote_endpoints: bool,
|
||||
/// First master address for volume lookups (e.g., "localhost:9333").
|
||||
/// First configured master address (e.g., "localhost:9333"). A seed, not a
|
||||
/// lookup target: lookups go to `current_master_url`.
|
||||
pub master_url: String,
|
||||
/// Seed master addresses for UI rendering.
|
||||
pub master_urls: Vec<String>,
|
||||
@@ -92,10 +95,12 @@ pub struct VolumeServerState {
|
||||
/// Built once at construction so Ping admission stays O(1). Mirrors
|
||||
/// Go's `seedMasterSet` on `VolumeServer`.
|
||||
pub seed_master_set: std::collections::HashSet<String>,
|
||||
/// Current master this server is heartbeating with, in canonical http
|
||||
/// `host:port` form. Empty when no heartbeat connection is active. The
|
||||
/// heartbeat goroutine writes; admission reads — the lock keeps them
|
||||
/// from racing on a leader change. Mirrors Go's `currentMaster` plus
|
||||
/// Master this server takes as leader, in its announced
|
||||
/// `host:port[.grpcPort]` form — callers pick http or gRPC form per use.
|
||||
/// Empty until the heartbeat confirms one — a peer that answered a
|
||||
/// stream without naming a leader, or the leader a follower named — and
|
||||
/// cleared again when no master reports one. The heartbeat writes;
|
||||
/// admission and lookups read. Mirrors Go's `currentMaster` plus
|
||||
/// `currentMasterLock`.
|
||||
pub current_master_url: tokio::sync::RwLock<String>,
|
||||
/// This server's own address (ip:port) for filtering self from lookup results.
|
||||
@@ -138,6 +143,13 @@ pub struct VolumeServerState {
|
||||
}
|
||||
|
||||
impl VolumeServerState {
|
||||
/// The master lookups should go to: the leader the heartbeat confirmed,
|
||||
/// or empty while none is known, in its announced `host:port[.grpcPort]`
|
||||
/// form. Mirrors Go's `vs.getCurrentMaster()`.
|
||||
pub async fn current_master(&self) -> String {
|
||||
self.current_master_url.read().await.clone()
|
||||
}
|
||||
|
||||
/// Check if the server is in maintenance mode; return gRPC error if so.
|
||||
pub fn check_maintenance(&self) -> Result<(), tonic::Status> {
|
||||
if self.maintenance.load(Ordering::Relaxed) {
|
||||
@@ -207,7 +219,7 @@ impl VolumeServerState {
|
||||
return false;
|
||||
}
|
||||
let current = self.current_master_url.read().await.clone();
|
||||
if !current.is_empty() && current == key {
|
||||
if !current.is_empty() && to_http_address(¤t) == key {
|
||||
return true;
|
||||
}
|
||||
self.seed_master_set.contains(&key)
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! Async batched write processing for the volume server.
|
||||
//!
|
||||
//! Instead of each upload handler directly calling `write_needle`, writes are
|
||||
//! submitted to a queue. Each volume gets its own FIFO worker (Go's
|
||||
//! Durable (fsync) uploads are submitted here instead of each handler calling
|
||||
//! `write_needle` directly — Go's `writeNeedle2` makes the same split, so the
|
||||
//! queue is always constructed. Each volume gets its own FIFO worker (Go's
|
||||
//! volume_write.go keeps one batching worker per volume), which drains its
|
||||
//! channel in batches of up to 128 entries or 4 MiB and shares the .dat/.idx
|
||||
//! flushes across durable writes (see `Volume::write_needles_grouped`).
|
||||
@@ -66,6 +67,9 @@ pub struct WriteQueue {
|
||||
/// alone would multiply the old single queue's capacity by volume count.
|
||||
queue_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
/// Submissions accepted, for tests asserting which writes route here.
|
||||
#[cfg(test)]
|
||||
submits: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl WriteQueue {
|
||||
@@ -78,9 +82,17 @@ impl WriteQueue {
|
||||
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
|
||||
queue_permits: Arc::new(Semaphore::new(capacity)),
|
||||
workers: Arc::new(Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
submits: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many writes were submitted, for tests asserting upload routing.
|
||||
#[cfg(test)]
|
||||
pub fn submits_for_test(&self) -> u64 {
|
||||
self.submits.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Sender for the volume's worker, spawning one on first use.
|
||||
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
|
||||
let mut workers = self.workers.lock().unwrap();
|
||||
@@ -110,6 +122,9 @@ impl WriteQueue {
|
||||
pub async fn submit(&self, volume_id: VolumeId, needle: Needle, fsync: bool) -> WriteResult {
|
||||
// Workers spawn only for mounted volumes — a bogus id never creates
|
||||
// a channel, so volume-id sprays can't grow the worker map.
|
||||
#[cfg(test)]
|
||||
self.submits
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
{
|
||||
let store = self.state.store.read().unwrap();
|
||||
if store.find_volume(volume_id).is_none() {
|
||||
|
||||
@@ -11,9 +11,11 @@ pub mod store_ec_mirror;
|
||||
pub mod store_ec_reconcile;
|
||||
pub mod super_block;
|
||||
pub mod types;
|
||||
pub mod vid_stamp;
|
||||
pub mod volume;
|
||||
pub mod volume_idx_rebuild;
|
||||
pub mod volume_idx_repair;
|
||||
pub mod volume_open;
|
||||
pub mod volume_report;
|
||||
pub mod volume_report_hash;
|
||||
pub mod volume_tail_repair;
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::config::MinFreeSpace;
|
||||
@@ -14,7 +14,7 @@ use crate::pb::master_pb;
|
||||
use crate::storage::disk_location::DiskLocation;
|
||||
use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
|
||||
use crate::storage::erasure_coding::ec_volume::{
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs,
|
||||
};
|
||||
use crate::storage::needle::needle::{Needle, get_actual_size};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -72,6 +72,10 @@ pub struct Store {
|
||||
/// reporting the heartbeat that saw it.
|
||||
pub reported_collections: Mutex<HashSet<String>>,
|
||||
pub reported_ec_collections: Mutex<HashSet<String>>,
|
||||
pub max_volume_id_seen: AtomicU32,
|
||||
pub last_seen_topology_id: Mutex<String>,
|
||||
pub vid_stamp_lock: Mutex<()>,
|
||||
pub vid_stamp_dirty: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -91,6 +95,10 @@ impl Store {
|
||||
rack: String::new(),
|
||||
reported_collections: Mutex::new(HashSet::new()),
|
||||
reported_ec_collections: Mutex::new(HashSet::new()),
|
||||
max_volume_id_seen: AtomicU32::new(0),
|
||||
last_seen_topology_id: Mutex::new(String::new()),
|
||||
vid_stamp_lock: Mutex::new(()),
|
||||
vid_stamp_dirty: std::sync::atomic::AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,6 +160,12 @@ impl Store {
|
||||
// (read-only target, partial copy).
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
|
||||
self.fold_location_into_vid_stamp(self.locations.last().unwrap());
|
||||
// Write the combined bound back to every dir, including ones whose
|
||||
// own stamp or volumes sit below it — a lost disk must not restart
|
||||
// the server below the bound its remaining dirs once reached.
|
||||
self.save_vid_stamps();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -402,6 +416,39 @@ impl Store {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Look up the per-volume EC data-shard count, preferring a mounted EC
|
||||
/// volume's own config and then the `.vif` on whichever disk holds it.
|
||||
/// Falls back to the default when neither exists yet (e.g. before the
|
||||
/// first shard streams in during ec.rebuild), so
|
||||
/// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio
|
||||
/// volumes against their real width instead of the build default.
|
||||
pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
for loc in &self.locations {
|
||||
if let Some(ec_vol) = loc.find_ec_volume(vid) {
|
||||
return ec_vol.data_shards;
|
||||
}
|
||||
}
|
||||
let Some(first) = self.locations.first() else {
|
||||
return DATA_SHARDS_COUNT as u32;
|
||||
};
|
||||
let other_dirs: Vec<String> = self.locations[1..]
|
||||
.iter()
|
||||
.flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()])
|
||||
.filter(|d| !d.is_empty())
|
||||
.collect();
|
||||
read_ec_shard_config_across_dirs(
|
||||
&first.directory,
|
||||
&first.idx_directory,
|
||||
&other_dirs,
|
||||
collection,
|
||||
vid,
|
||||
)
|
||||
.map(|(data_shards, _, _)| data_shards)
|
||||
.unwrap_or(DATA_SHARDS_COUNT as u32)
|
||||
}
|
||||
|
||||
/// Create a new volume, placing it on the location with the most free space.
|
||||
pub fn add_volume(
|
||||
&mut self,
|
||||
@@ -409,6 +456,7 @@ impl Store {
|
||||
disk_type: DiskType,
|
||||
spec: &VolumeSpec<'_>,
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
if self.find_volume(vid).is_some() {
|
||||
return Err(VolumeError::AlreadyExists);
|
||||
}
|
||||
@@ -506,7 +554,7 @@ impl Store {
|
||||
vid
|
||||
))));
|
||||
}
|
||||
return loc.create_volume(
|
||||
let result = loc.create_volume(
|
||||
vid,
|
||||
self.needle_map_kind,
|
||||
&VolumeSpec {
|
||||
@@ -514,6 +562,10 @@ impl Store {
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
if result.is_ok() {
|
||||
self.stamp_allocated_vid(vid);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Err(VolumeError::Io(io::Error::new(
|
||||
@@ -603,7 +655,10 @@ impl Store {
|
||||
..Default::default()
|
||||
},
|
||||
) {
|
||||
Ok(()) => return Ok(()),
|
||||
Ok(()) => {
|
||||
self.stamp_allocated_vid(vid);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
continue;
|
||||
@@ -659,7 +714,10 @@ impl Store {
|
||||
..Default::default()
|
||||
},
|
||||
) {
|
||||
Ok(()) => return Ok(()),
|
||||
Ok(()) => {
|
||||
self.stamp_allocated_vid(vid);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
continue;
|
||||
@@ -952,6 +1010,7 @@ impl Store {
|
||||
collection: &str,
|
||||
shard_ids: &[ShardId],
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
// Find the location where the EC files live
|
||||
let loc_idx = self.find_ec_location(vid, collection).ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
@@ -974,6 +1033,7 @@ impl Store {
|
||||
shard_id: ShardId,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
// The .ecx may live on a different disk than the shard being mounted
|
||||
// (ec.balance / ec.rebuild can spread shards across sibling disks), so
|
||||
// look up its owner once and point EcVolume::new at the directory that
|
||||
@@ -2953,6 +3013,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A volume whose `.vif` declares a custom EC ratio must feed its own
|
||||
/// data-shard count into the free-slot maths — not the build default —
|
||||
/// wherever the `.vif` happens to sit. With no `.vif` anywhere (a
|
||||
/// brand-new volume mid-ec.rebuild) the answer is the default.
|
||||
#[test]
|
||||
fn test_lookup_ec_data_shards_reads_vif_on_any_disk() {
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
|
||||
let (store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "custom-ratio";
|
||||
let vid = VolumeId(4242);
|
||||
|
||||
assert_eq!(
|
||||
store.lookup_ec_data_shards(collection, vid),
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
"no .vif anywhere must answer the default"
|
||||
);
|
||||
|
||||
// A 6+2 .vif lands on disk 2 only — the rebuild staging disk.
|
||||
let base = volume_file_name(&store.locations[2].directory, collection, vid);
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(store.lookup_ec_data_shards(collection, vid), 6);
|
||||
}
|
||||
|
||||
/// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`;
|
||||
/// a disk this delete never changed keeps its sidecar. Disk 0 models
|
||||
/// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
//! Persisted volume-id high-water mark ("vid stamp"), mirroring Go's
|
||||
//! `weed/storage/vid_stamp.go`.
|
||||
//!
|
||||
//! Each volume dir carries a `.vid_stamp` JSON file with the highest volume
|
||||
//! id the server has ever observed plus the topology id it last heartbeated
|
||||
//! to. The bound is fed by heartbeat echoes, AllocateVolume, and local
|
||||
//! volume/EC creation, is reported back in `Heartbeat.max_volume_id_seen`,
|
||||
//! and survives master state loss so rebuilt masters cannot reissue ids an
|
||||
//! offline server still holds.
|
||||
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
use super::store::Store;
|
||||
|
||||
const VID_STAMP_FILE_NAME: &str = ".vid_stamp";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
struct VidStamp {
|
||||
#[serde(rename = "maxVolumeId")]
|
||||
max_volume_id: u32,
|
||||
#[serde(rename = "topologyId", default, skip_serializing_if = "String::is_empty")]
|
||||
topology_id: String,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn max_volume_id_seen(&self) -> u32 {
|
||||
self.max_volume_id_seen.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn last_seen_topology_id(&self) -> String {
|
||||
self.last_seen_topology_id
|
||||
.lock()
|
||||
.unwrap()
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn update_max_volume_id_seen(&self, vid: u32) {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
let prev = self.max_volume_id_seen.fetch_max(vid, Relaxed);
|
||||
if vid > prev || self.vid_stamp_dirty.load(Relaxed) {
|
||||
self.save_vid_stamps();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_last_seen_topology_id(&self, id: &str) {
|
||||
if id.is_empty() {
|
||||
return;
|
||||
}
|
||||
{
|
||||
let mut guard = self.last_seen_topology_id.lock().unwrap();
|
||||
if *guard == id && !self.vid_stamp_dirty.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
*guard = id.to_string();
|
||||
}
|
||||
self.save_vid_stamps();
|
||||
}
|
||||
|
||||
/// Seed the bound from a freshly added location: its persisted stamp,
|
||||
/// its volumes, and its EC shards. Called per location from
|
||||
/// `add_location` after the EC reconcile passes have run.
|
||||
pub(crate) fn fold_location_into_vid_stamp(&self, loc: &super::disk_location::DiskLocation) {
|
||||
if let Ok(stamp) = read_vid_stamp(Path::new(&loc.directory)) {
|
||||
self.update_max_volume_id_seen(stamp.max_volume_id);
|
||||
if !stamp.topology_id.is_empty() {
|
||||
self.update_last_seen_topology_id(&stamp.topology_id);
|
||||
}
|
||||
}
|
||||
let mut max_vid: u32 = 0;
|
||||
for vid in loc.volume_ids() {
|
||||
max_vid = max_vid.max(vid.0);
|
||||
}
|
||||
for (vid, _) in loc.ec_volumes() {
|
||||
max_vid = max_vid.max(vid.0);
|
||||
}
|
||||
if max_vid > 0 {
|
||||
self.update_max_volume_id_seen(max_vid);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn save_vid_stamps(&self) {
|
||||
let _guard = self.vid_stamp_lock.lock().unwrap();
|
||||
let stamp = match serde_json::to_vec(&VidStamp {
|
||||
max_volume_id: self.max_volume_id_seen(),
|
||||
topology_id: self.last_seen_topology_id(),
|
||||
}) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut failed = false;
|
||||
for loc in &self.locations {
|
||||
if let Err(e) = write_vid_stamp(Path::new(&loc.directory), &stamp) {
|
||||
tracing::warn!("write vid stamp in {}: {}", loc.directory, e);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
self.vid_stamp_dirty
|
||||
.store(failed, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn read_vid_stamp(dir: &Path) -> io::Result<VidStamp> {
|
||||
let data = fs::read(dir.join(VID_STAMP_FILE_NAME))?;
|
||||
serde_json::from_slice(&data).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
fn write_vid_stamp(dir: &Path, stamp: &[u8]) -> io::Result<()> {
|
||||
let tmp = dir.join(format!("{}.tmp", VID_STAMP_FILE_NAME));
|
||||
{
|
||||
use std::io::Write;
|
||||
let mut f = fs::File::create(&tmp)?;
|
||||
f.write_all(stamp)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
fs::rename(&tmp, dir.join(VID_STAMP_FILE_NAME))?;
|
||||
if cfg!(unix) {
|
||||
fs::File::open(dir)?.sync_all()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stamps the volume id of a newly created volume or EC shard set. Called
|
||||
/// from `Store::add_volume` / `Store::mount_ec_shards*` so the bound covers
|
||||
/// ids before their files exist.
|
||||
impl Store {
|
||||
pub(crate) fn stamp_allocated_vid(&self, vid: VolumeId) {
|
||||
self.update_max_volume_id_seen(vid.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::types::DiskType;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_store(dir: &str) -> Store {
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(dir, dir, 10, DiskType::HardDrive, MinFreeSpace::Percent(1.0), Vec::new())
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stamp_written_on_bound_increase() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let store = make_store(dir);
|
||||
|
||||
store.update_max_volume_id_seen(42);
|
||||
|
||||
let stamp = read_vid_stamp(tmp.path()).unwrap();
|
||||
assert_eq!(stamp.max_volume_id, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stamp_survives_restart() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let store = make_store(dir);
|
||||
store.update_max_volume_id_seen(42);
|
||||
store.update_last_seen_topology_id("cluster-a");
|
||||
drop(store);
|
||||
|
||||
let restarted = make_store(dir);
|
||||
assert_eq!(restarted.max_volume_id_seen(), 42);
|
||||
assert_eq!(restarted.last_seen_topology_id(), "cluster-a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn later_disk_gets_combined_stamp() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
// dir2 carries a stale stamp from an older run
|
||||
write_vid_stamp(
|
||||
tmp2.path(),
|
||||
&serde_json::to_vec(&VidStamp {
|
||||
max_volume_id: 10,
|
||||
topology_id: String::new(),
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
tmp1.path().to_str().unwrap(),
|
||||
tmp1.path().to_str().unwrap(),
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store.update_max_volume_id_seen(100);
|
||||
store
|
||||
.add_location(
|
||||
tmp2.path().to_str().unwrap(),
|
||||
tmp2.path().to_str().unwrap(),
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let stamp2 = read_vid_stamp(tmp2.path()).unwrap();
|
||||
assert_eq!(stamp2.max_volume_id, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_is_monotonic() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let store = make_store(dir);
|
||||
|
||||
store.update_max_volume_id_seen(10);
|
||||
store.update_max_volume_id_seen(5);
|
||||
assert_eq!(store.max_volume_id_seen(), 10);
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, Red
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE, SuperBlock};
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume_open::open_volume_file;
|
||||
use crate::storage::volume_tail_repair::TailRepairError;
|
||||
|
||||
// ============================================================================
|
||||
// Errors
|
||||
@@ -1247,6 +1248,12 @@ pub struct Volume {
|
||||
idx_syncs_for_test: std::sync::atomic::AtomicUsize,
|
||||
needle_map_kind: NeedleMapKind,
|
||||
data_file_access_control: Arc<DataFileAccessControl>,
|
||||
/// Serializes a data-file mutation with the sync that makes it durable:
|
||||
/// write, delete and lifecycle paths hold it for their whole critical
|
||||
/// section, while the fsync itself runs with only the `io` read guard so
|
||||
/// readers of this volume do not wait on it. Lock order is
|
||||
/// `data_file_sync_lock` -> `data_file_access_control` -> `io`.
|
||||
data_file_sync_lock: std::sync::Mutex<()>,
|
||||
|
||||
/// Go's superBlockAccessLock.
|
||||
super_block: RwLock<SuperBlock>,
|
||||
@@ -1370,6 +1377,7 @@ impl Volume {
|
||||
idx_syncs_for_test: Default::default(),
|
||||
needle_map_kind,
|
||||
data_file_access_control: Arc::new(DataFileAccessControl::default()),
|
||||
data_file_sync_lock: std::sync::Mutex::new(()),
|
||||
super_block: RwLock::new(SuperBlock {
|
||||
replica_placement: replica_placement.unwrap_or_default(),
|
||||
ttl: ttl.unwrap_or(crate::storage::needle::ttl::TTL::EMPTY),
|
||||
@@ -1426,6 +1434,7 @@ impl Volume {
|
||||
idx_syncs_for_test: Default::default(),
|
||||
needle_map_kind: NeedleMapKind::InMemory,
|
||||
data_file_access_control: Arc::new(DataFileAccessControl::default()),
|
||||
data_file_sync_lock: std::sync::Mutex::new(()),
|
||||
super_block: RwLock::new(SuperBlock::default()),
|
||||
no_write: AtomicU8::new(0),
|
||||
io_unavailable: Arc::default(),
|
||||
@@ -1446,6 +1455,13 @@ impl Volume {
|
||||
self.data_file_access_control.clone()
|
||||
}
|
||||
|
||||
/// The mutation/sync serializer, for callers that manage the access
|
||||
/// lease themselves (the `_held` lifecycle paths). Same lock order as
|
||||
/// everywhere else: sync lock first, then the write lease, then `io`.
|
||||
pub(crate) fn data_file_sync_lock(&self) -> &std::sync::Mutex<()> {
|
||||
&self.data_file_sync_lock
|
||||
}
|
||||
|
||||
pub(crate) fn is_instance(&self, instance: &Arc<DataFileAccessControl>) -> bool {
|
||||
Arc::ptr_eq(instance, &self.data_file_access_control)
|
||||
}
|
||||
@@ -1709,6 +1725,48 @@ impl Volume {
|
||||
),
|
||||
}
|
||||
|
||||
// A crash between the .dat append and the .idx append leaves a
|
||||
// .dat tail no index entry covers. Re-index complete needles and
|
||||
// cut an incomplete remainder before the index loads, so the
|
||||
// recovered rows land in the needle map and inside its append
|
||||
// offset. Mirrors Go running repairUnindexedDatTail inside
|
||||
// CheckVolumeDataIntegrity ahead of the needle map load. Only a
|
||||
// volume about to load writable is repaired.
|
||||
// A no_write_can_delete volume still appends tombstone records on
|
||||
// delete, so it tears tails the same way and gets the same repair.
|
||||
if !self.has_remote_file() && !self.no_write_flags().0 {
|
||||
// An empty .idx gets the same repair: every check below
|
||||
// skips an empty index on its own, so a tail the repair
|
||||
// cannot explain there fails the load like in Go rather
|
||||
// than warn and load writable.
|
||||
let idx_empty = fs::metadata(self.file_name(".idx"))
|
||||
.map(|m| m.len() == 0)
|
||||
.unwrap_or(false);
|
||||
match self.repair_unindexed_dat_tail(io) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.is_not_torn() && !idx_empty => warn!(
|
||||
volume_id = self.id.0,
|
||||
error = %e,
|
||||
"unindexed .dat tail not repaired"
|
||||
),
|
||||
// The index refusing a read-write open is the same failure
|
||||
// Go reports opening it ahead of the check: it fails the
|
||||
// load rather than only the repair.
|
||||
Err(TailRepairError::OpenIndex(e)) => return Err(e),
|
||||
Err(e) => {
|
||||
// A repair that could not complete must not load the
|
||||
// volume writable on files it half-fixed: the check
|
||||
// fails as in Go, so the volume loads read-only.
|
||||
self.set_no_write_flag(NO_WRITE_OR_DELETE, true);
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
error = %e,
|
||||
"repair unindexed .dat tail"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.load_index(io)?;
|
||||
|
||||
// Match Go: CheckVolumeDataIntegrity after loading index (volume_loading.go L154-159)
|
||||
@@ -1980,6 +2038,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
pub(crate) fn load_remote_dat_file(&self) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
self.load_remote_dat_file_io(&mut io)
|
||||
@@ -2066,7 +2125,7 @@ impl Volume {
|
||||
self.current_dat_file_size_io(&io)
|
||||
}
|
||||
|
||||
fn current_dat_file_size_io(&self, io: &VolumeIo) -> io::Result<u64> {
|
||||
pub(crate) fn current_dat_file_size_io(&self, io: &VolumeIo) -> io::Result<u64> {
|
||||
if let Some(ref f) = io.dat_file {
|
||||
Ok(f.metadata()?.len())
|
||||
} else if let Some(ref remote_dat_file) = io.remote_dat_file {
|
||||
@@ -2290,7 +2349,7 @@ impl Volume {
|
||||
self.read_needle_blob_unlocked(&io, offset, size)
|
||||
}
|
||||
|
||||
fn read_needle_blob_unlocked(
|
||||
pub(crate) fn read_needle_blob_unlocked(
|
||||
&self,
|
||||
io: &VolumeIo,
|
||||
offset: i64,
|
||||
@@ -2509,10 +2568,14 @@ impl Volume {
|
||||
check_cookie: bool,
|
||||
fsync: bool,
|
||||
) -> Result<(u64, Size, bool), VolumeError> {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
self.check_writable()?;
|
||||
self.do_write_request(&mut io, n, check_cookie, fsync)
|
||||
// A single write is a one-entry run of the grouped path: identical
|
||||
// outcome, and the fsync handling lives in one place.
|
||||
debug_assert!(check_cookie, "every write_needle caller checks the cookie");
|
||||
let _ = check_cookie;
|
||||
let mut writes = [(std::mem::take(n), fsync)];
|
||||
let mut results = self.write_needles_grouped(&mut writes);
|
||||
*n = std::mem::replace(&mut writes[0].0, Needle::default());
|
||||
results.pop().unwrap_or(Err(VolumeError::NotFound))
|
||||
}
|
||||
|
||||
/// Write a batch of needles the way Go's processBatch does: one .dat
|
||||
@@ -2527,8 +2590,13 @@ impl Volume {
|
||||
&self,
|
||||
writes: &mut [(Needle, bool)],
|
||||
) -> Vec<Result<(u64, Size, bool), VolumeError>> {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
// The sync lock serializes a .dat mutation with the flush that makes
|
||||
// it durable: a failed sync's rollback can never clobber an append
|
||||
// that slipped in behind it, and a compaction swap or unlink cannot
|
||||
// land between the append and its sync. Readers never take it, and
|
||||
// the write-side guards are dropped for the sync itself, so reads of
|
||||
// this volume do not wait on fsync.
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let mut results = Vec::with_capacity(writes.len());
|
||||
let mut rest = writes;
|
||||
while !rest.is_empty() {
|
||||
@@ -2536,8 +2604,10 @@ impl Volume {
|
||||
let len = rest.iter().take_while(|(n, _)| ids.insert(n.id)).count();
|
||||
let (run, tail) = std::mem::take(&mut rest).split_at_mut(len);
|
||||
if run.iter().any(|(_, fsync)| *fsync) {
|
||||
results.extend(self.write_synced_run(&mut io, run));
|
||||
results.extend(self.write_synced_run(run));
|
||||
} else {
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
for (n, _) in run.iter_mut() {
|
||||
results.push(
|
||||
self.check_writable()
|
||||
@@ -2552,7 +2622,6 @@ impl Volume {
|
||||
|
||||
fn write_synced_run(
|
||||
&self,
|
||||
io: &mut VolumeIo,
|
||||
run: &mut [(Needle, bool)],
|
||||
) -> Vec<Result<(u64, Size, bool), VolumeError>> {
|
||||
// Per entry: Some(offset) once appended, None when it dedups.
|
||||
@@ -2560,63 +2629,82 @@ impl Volume {
|
||||
// Per entry: the I/O error streak once it is staged, where a write
|
||||
// sent on its own would have recorded its success.
|
||||
let mut marks = Vec::with_capacity(run.len());
|
||||
let mut last_append_at_ns = io.last_append_at_ns;
|
||||
let mut last_append_at_ns;
|
||||
let mut run_start = None;
|
||||
let mut sync = false;
|
||||
for (n, fsync) in run.iter_mut() {
|
||||
let r = self.append_unpublished(io, n, &mut last_append_at_ns);
|
||||
marks.push(self.io_errors.mark());
|
||||
if let Ok(Some(offset)) = r {
|
||||
run_start.get_or_insert(offset);
|
||||
}
|
||||
// Only a durable entry that got this far needs the sync.
|
||||
sync |= *fsync && r.is_ok();
|
||||
staged.push(r);
|
||||
}
|
||||
|
||||
if sync && let Err(e) = self.flush_dat(io) {
|
||||
self.check_read_write_error(Some(&e));
|
||||
if let Some(start) = run_start {
|
||||
self.undo_unsynced_append(io, start);
|
||||
}
|
||||
let e = VolumeError::Io(e);
|
||||
return staged
|
||||
.into_iter()
|
||||
.map(|r| r.and_then(|_| Err(run_error(&e))))
|
||||
.collect();
|
||||
}
|
||||
io.last_append_at_ns = last_append_at_ns;
|
||||
for ((n, _), r) in run.iter().zip(&staged) {
|
||||
if matches!(r, Ok(Some(_))) && n.append_at_ns > io.last_write_append_at_ns {
|
||||
io.last_write_append_at_ns = n.append_at_ns;
|
||||
io.last_write_needle_key = n.id;
|
||||
io.last_write_deleted = false;
|
||||
{
|
||||
// Append under the write guards, then drop them for the flush:
|
||||
// readers need only `io`'s read side, while mutators and a
|
||||
// compaction swap stay excluded by the caller's sync lock.
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
last_append_at_ns = io.last_append_at_ns;
|
||||
for (n, fsync) in run.iter_mut() {
|
||||
let r = self.append_unpublished(&mut io, n, &mut last_append_at_ns);
|
||||
marks.push(self.io_errors.mark());
|
||||
if let Ok(Some(offset)) = r {
|
||||
run_start.get_or_insert(offset);
|
||||
}
|
||||
// Only a durable entry that got this far needs the sync.
|
||||
sync |= *fsync && r.is_ok();
|
||||
staged.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
// A durable entry that fails to publish stops the volume taking
|
||||
// writes, so the entries after it are refused the way a lone
|
||||
// write would be.
|
||||
let mut refused = false;
|
||||
for ((n, fsync), r) in run.iter().zip(staged.iter_mut()) {
|
||||
if refused {
|
||||
*r = Err(self
|
||||
.check_writable()
|
||||
.err()
|
||||
.unwrap_or(VolumeError::ReadOnly(self.id)));
|
||||
} else if let Ok(Some(offset)) = *r
|
||||
&& let Err(e) = self.publish_write(io, n, offset, *fsync)
|
||||
{
|
||||
*r = Err(e);
|
||||
refused = *fsync;
|
||||
if sync {
|
||||
let io = self.io.read().unwrap();
|
||||
if let Err(e) = self.flush_dat(&io) {
|
||||
self.check_read_write_error(Some(&e));
|
||||
if let Some(start) = run_start {
|
||||
self.undo_unsynced_append(&io, start);
|
||||
}
|
||||
let e = VolumeError::Io(e);
|
||||
return staged
|
||||
.into_iter()
|
||||
.map(|r| r.and_then(|_| Err(run_error(&e))))
|
||||
.collect();
|
||||
}
|
||||
}
|
||||
|
||||
if sync && let Err(e) = self.flush_idx(io) {
|
||||
return staged
|
||||
.into_iter()
|
||||
.map(|r| r.and_then(|_| Err(run_error(&e))))
|
||||
.collect();
|
||||
{
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
io.last_append_at_ns = last_append_at_ns;
|
||||
for ((n, _), r) in run.iter().zip(&staged) {
|
||||
if matches!(r, Ok(Some(_))) && n.append_at_ns > io.last_write_append_at_ns {
|
||||
io.last_write_append_at_ns = n.append_at_ns;
|
||||
io.last_write_needle_key = n.id;
|
||||
io.last_write_deleted = false;
|
||||
}
|
||||
}
|
||||
|
||||
// A durable entry that fails to publish stops the volume taking
|
||||
// writes, so the entries after it are refused the way a lone
|
||||
// write would be.
|
||||
let mut refused = false;
|
||||
for ((n, fsync), r) in run.iter().zip(staged.iter_mut()) {
|
||||
if refused {
|
||||
*r = Err(self
|
||||
.check_writable()
|
||||
.err()
|
||||
.unwrap_or(VolumeError::ReadOnly(self.id)));
|
||||
} else if let Ok(Some(offset)) = *r
|
||||
&& let Err(e) = self.publish_write(&mut io, n, offset, *fsync)
|
||||
{
|
||||
*r = Err(e);
|
||||
refused = *fsync;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sync {
|
||||
let io = self.io.read().unwrap();
|
||||
if let Err(e) = self.flush_idx(&io) {
|
||||
return staged
|
||||
.into_iter()
|
||||
.map(|r| r.and_then(|_| Err(run_error(&e))))
|
||||
.collect();
|
||||
}
|
||||
}
|
||||
|
||||
let written = run
|
||||
@@ -2630,7 +2718,9 @@ impl Volume {
|
||||
// Sent one at a time, the last write to land would have cleared
|
||||
// the streak of the appends before it, and the ones after it
|
||||
// would have failed on top of that.
|
||||
self.finish_write(io, last_modified, sync, marks[last]);
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
self.finish_write(&mut io, last_modified, sync, marks[last]);
|
||||
}
|
||||
|
||||
run.iter()
|
||||
@@ -2679,7 +2769,7 @@ impl Volume {
|
||||
/// Flush the .dat, the first half of a durable write. The .idx is flushed
|
||||
/// separately by flush_idx once the row is published; the two are split so
|
||||
/// nothing is indexed before the bytes it points at are down.
|
||||
fn flush_dat(&self, io: &VolumeIo) -> io::Result<()> {
|
||||
pub(crate) fn flush_dat(&self, io: &VolumeIo) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
self.dat_syncs_for_test.fetch_add(1, Ordering::Relaxed);
|
||||
#[cfg(test)]
|
||||
@@ -2731,7 +2821,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Take an unflushed append back off the .dat after its sync failed.
|
||||
fn truncate_dat(&self, io: &VolumeIo, len: u64) -> io::Result<()> {
|
||||
pub(crate) fn truncate_dat(&self, io: &VolumeIo, len: u64) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
if io.fail_truncate_for_test {
|
||||
return Err(io::Error::other("injected truncate failure"));
|
||||
@@ -3082,6 +3172,7 @@ impl Volume {
|
||||
|
||||
/// Delete a needle from the volume.
|
||||
pub fn delete_needle(&self, n: &mut Needle) -> Result<Size, VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
if self.is_closed() {
|
||||
@@ -3464,7 +3555,7 @@ impl Volume {
|
||||
/// key-sorted .idx (e.g. legacy weed fix output) falls back to a single
|
||||
/// linear scan for the maximum offset. Returns -1 when no entry has a
|
||||
/// non-zero offset (a pre-allocated / all-zero index).
|
||||
fn find_dat_tail_entry_offset(
|
||||
pub(crate) fn find_dat_tail_entry_offset(
|
||||
&self,
|
||||
io: &VolumeIo,
|
||||
idx_file: &mut File,
|
||||
@@ -3981,6 +4072,7 @@ impl Volume {
|
||||
offset: Offset,
|
||||
size: Size,
|
||||
) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let mut io = self.io.write().unwrap();
|
||||
if let Some(ref mut nm) = io.nm {
|
||||
@@ -4005,6 +4097,7 @@ impl Volume {
|
||||
// The flag flip and the .vif save are one critical section with the idx
|
||||
// attach: Go's markVolumeReadonly runs under dataFileAccessLock, so a
|
||||
// racing mark-writable cannot interleave a save of half-applied state.
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
self.set_read_only_persist_held(can_delete, persist)
|
||||
}
|
||||
@@ -4089,10 +4182,13 @@ impl Volume {
|
||||
if let Some(e) = self.unavailable_error() {
|
||||
return Err(e);
|
||||
}
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let (was_no_write_or_delete, was_no_write_can_delete) = self.no_write_flags();
|
||||
// Remote-tiered volumes must stay no_write_can_delete regardless of marks.
|
||||
let no_write_can_delete = was_no_write_can_delete && self.has_remote_file();
|
||||
// A remote-tiered volume keeps the delete-only mode a load derives: its
|
||||
// .dat cannot take appends, so "writable" resolves to can-delete rather
|
||||
// than advertising writable with no local data file.
|
||||
let no_write_can_delete = self.has_remote_file();
|
||||
self.no_write.store(
|
||||
if no_write_can_delete {
|
||||
NO_WRITE_CAN_DELETE
|
||||
@@ -4187,6 +4283,7 @@ impl Volume {
|
||||
&self,
|
||||
f: impl FnOnce(&mut Vec<PbRemoteFile>),
|
||||
) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
self.update_remote_files_held(f)
|
||||
}
|
||||
@@ -4205,21 +4302,70 @@ impl Volume {
|
||||
}
|
||||
f(&mut self.volume_info.write().unwrap().files);
|
||||
let mut io = self.io.write().unwrap();
|
||||
self.refresh_remote_write_mode(&mut io)
|
||||
self.refresh_remote_write_mode(&mut io, true)
|
||||
}
|
||||
|
||||
/// Go's MarkTieredDeletable: a live tier-up lifts the upload-time
|
||||
/// read-only mark, except on a replicated volume — the shell copies the
|
||||
/// uploaded replica's .idx and .vif to the other replicas after the
|
||||
/// upload, and a tombstone written before that copy would reach one
|
||||
/// replica only — or under an I/O quarantine, which is not the tier
|
||||
/// upload's to lift. A refused volume keeps its flags until the next
|
||||
/// load, as the Go volume does.
|
||||
fn mark_tiered_deletable(&self) -> bool {
|
||||
if self
|
||||
.super_block
|
||||
.read()
|
||||
.unwrap()
|
||||
.replica_placement
|
||||
.has_replication()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.has_remote_file() {
|
||||
return false;
|
||||
}
|
||||
// Go takes noWriteLock around the quarantine check and the flag
|
||||
// change; here a compare-exchange gives the same ordering — a
|
||||
// heartbeat refusal landing between the check and the swap fails the
|
||||
// swap, and the retry sees the quarantine mark the heartbeat set
|
||||
// first.
|
||||
loop {
|
||||
if self.should_quarantine() {
|
||||
return false;
|
||||
}
|
||||
let flags = self.no_write.load(Ordering::Acquire);
|
||||
let promoted = (flags & !NO_WRITE_OR_DELETE) | NO_WRITE_CAN_DELETE;
|
||||
if self
|
||||
.no_write
|
||||
.compare_exchange(flags, promoted, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Recompute the Go-style write/delete mode from the current remote tier
|
||||
/// state, and bring the needle map in line with it — a volume that stops
|
||||
/// being remote also stops using the read-only sorted index.
|
||||
///
|
||||
/// `live` is a tier transition on a running volume: the promotion then
|
||||
/// goes through mark_tiered_deletable, Go's MarkTieredDeletable, while a
|
||||
/// load promotes every tiered volume unconditionally as Go's load() does.
|
||||
///
|
||||
/// If the map cannot be rebuilt the volume is pinned read-only rather than
|
||||
/// published as writable with an index that rejects every put; a restart
|
||||
/// recovers it.
|
||||
fn refresh_remote_write_mode(&self, io: &mut VolumeIo) -> Result<(), VolumeError> {
|
||||
fn refresh_remote_write_mode(&self, io: &mut VolumeIo, live: bool) -> Result<(), VolumeError> {
|
||||
let has_remote = self.has_remote_file();
|
||||
if has_remote {
|
||||
// remote-tiered: no writes, deletes allowed
|
||||
self.no_write.store(NO_WRITE_CAN_DELETE, Ordering::Release);
|
||||
if !live {
|
||||
self.no_write.store(NO_WRITE_CAN_DELETE, Ordering::Release);
|
||||
} else {
|
||||
self.mark_tiered_deletable();
|
||||
}
|
||||
} else if !self.volume_info.read().unwrap().read_only_can_delete {
|
||||
// only clear the remoteness-derived flag, not an operator mark
|
||||
self.set_no_write_flag(NO_WRITE_CAN_DELETE, false);
|
||||
@@ -4268,7 +4414,7 @@ impl Volume {
|
||||
}
|
||||
}
|
||||
*self.volume_info.write().unwrap() = pb_info;
|
||||
self.refresh_remote_write_mode(io)?;
|
||||
self.refresh_remote_write_mode(io, false)?;
|
||||
{
|
||||
let mut vi = self.volume_info.write().unwrap();
|
||||
if vi.version == 0 {
|
||||
@@ -4299,7 +4445,7 @@ impl Volume {
|
||||
self.set_no_write_flag(NO_WRITE_OR_DELETE, true);
|
||||
}
|
||||
*self.volume_info.write().unwrap() = pb_info;
|
||||
self.refresh_remote_write_mode(io)?;
|
||||
self.refresh_remote_write_mode(io, false)?;
|
||||
{
|
||||
let mut vi = self.volume_info.write().unwrap();
|
||||
if vi.version == 0 {
|
||||
@@ -4451,6 +4597,7 @@ impl Volume {
|
||||
|
||||
/// Change the replication placement and rewrite the super block.
|
||||
pub fn set_replica_placement(&self, rp: ReplicaPlacement) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
let bytes = {
|
||||
let mut sb = self.super_block.write().unwrap();
|
||||
@@ -4621,6 +4768,7 @@ impl Volume {
|
||||
|
||||
/// Write a raw needle blob at a specific offset in the .dat file.
|
||||
pub fn write_needle_blob(&self, offset: i64, needle_blob: &[u8]) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
if self.is_read_only() {
|
||||
return Err(VolumeError::ReadOnly(self.id));
|
||||
@@ -4652,6 +4800,7 @@ impl Volume {
|
||||
needle_blob: &[u8],
|
||||
size: Size,
|
||||
) -> Result<(), VolumeError> {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
// nm.put on a read-only volume fails only after the blob is appended to .dat.
|
||||
if self.is_read_only() {
|
||||
@@ -4956,6 +5105,16 @@ impl Volume {
|
||||
// would stall every volume behind one slow client. The vacuum-commit
|
||||
// RPC uses the lease-holding path instead — it waits with no store
|
||||
// guard held, matching Go's blocking commit.
|
||||
// The sync lock must come before the lease and be non-blocking like
|
||||
// it: a durable write holds it across its fsync, and the commit must
|
||||
// not swap files over an unpublished write — nor stall the store
|
||||
// behind one.
|
||||
let Ok(_sync_guard) = self.data_file_sync_lock.try_lock() else {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
COMMIT_WAITS_FOR_READS,
|
||||
)));
|
||||
};
|
||||
let Some(_lease) = self.data_file_access_control.try_write_lock() else {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
@@ -4965,7 +5124,9 @@ impl Volume {
|
||||
self.commit_compact_held()
|
||||
}
|
||||
|
||||
/// `commit_compact` for callers already holding the write lease.
|
||||
/// `commit_compact` for callers already holding `data_file_sync_lock`
|
||||
/// and the write lease — the swap must not land between a durable
|
||||
/// write's append and its publish.
|
||||
pub(crate) fn commit_compact_held(&self) -> Result<(), VolumeError> {
|
||||
// A handle detached between lookup and lease must not swap compacted
|
||||
// files over the ones its remounted replacement now owns.
|
||||
@@ -5356,6 +5517,7 @@ impl Volume {
|
||||
/// Go's `Volume.Close`: the write lease excludes in-flight I/O — with a
|
||||
/// shared handle, `&mut self` no longer provides that exclusion.
|
||||
pub fn close(&self) {
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
self.closed.store(true, Ordering::Release);
|
||||
self.close_locked();
|
||||
@@ -5401,6 +5563,7 @@ impl Volume {
|
||||
if self.is_compacting() {
|
||||
return Err(self.compacting_error());
|
||||
}
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
|
||||
if self.dir_idx() == new_idx_dir {
|
||||
@@ -5463,6 +5626,7 @@ impl Volume {
|
||||
) -> Result<(), VolumeError> {
|
||||
// Go's Destroy holds dataFileAccessLock: file unlinking must exclude
|
||||
// in-flight I/O from a shared handle, not just borrow exclusivity.
|
||||
let _sync_guard = self.data_file_sync_lock.lock().unwrap();
|
||||
let _guard = self.data_file_access_control.write_lock();
|
||||
// Either enabled check may pass: a volume with no live data qualifies
|
||||
// whether it reads empty or as all garbage. Byte counters, not counts:
|
||||
@@ -9588,6 +9752,72 @@ mod tests {
|
||||
assert!(!v.is_no_write_or_delete());
|
||||
}
|
||||
|
||||
/// Go's MarkTieredDeletable leaves a replicated volume's upload-time
|
||||
/// read-only mark alone: the shell still has to copy the uploaded
|
||||
/// replica's .idx to the others, and a tombstone written before that copy
|
||||
/// would reach one replica only.
|
||||
#[test]
|
||||
fn test_update_remote_files_keeps_a_replicated_volume_refusing_deletes() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
v.super_block.write().unwrap().replica_placement =
|
||||
crate::storage::super_block::ReplicaPlacement::from_string("001").unwrap();
|
||||
v.set_no_write_or_delete(true); // the shell's pre-upload mark
|
||||
|
||||
v.update_remote_files(|files| files.push(Default::default()))
|
||||
.unwrap();
|
||||
|
||||
assert!(v.has_remote_file());
|
||||
assert!(
|
||||
v.is_no_write_or_delete(),
|
||||
"a replicated volume keeps refusing deletes until its next load"
|
||||
);
|
||||
assert!(!v.is_no_write_can_delete());
|
||||
}
|
||||
|
||||
/// An I/O quarantine is not the tier upload's to lift: a live tier-up
|
||||
/// leaves a quarantined volume's read-only mark alone, as Go's
|
||||
/// MarkTieredDeletable does.
|
||||
#[test]
|
||||
fn test_update_remote_files_keeps_an_io_quarantine() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
v.set_no_write_or_delete(true);
|
||||
v.mark_io_quarantined();
|
||||
|
||||
v.update_remote_files(|files| files.push(Default::default()))
|
||||
.unwrap();
|
||||
|
||||
assert!(v.has_remote_file());
|
||||
assert!(v.is_no_write_or_delete());
|
||||
assert!(!v.is_no_write_can_delete());
|
||||
}
|
||||
|
||||
/// A refused tiered volume keeps OR_DELETE without CAN_DELETE; marking it
|
||||
/// writable must resolve to the delete-only mode a load derives — the
|
||||
/// remote .dat cannot take appends — not drop every protection.
|
||||
#[test]
|
||||
fn test_set_writable_on_a_refused_tiered_volume_stays_delete_only() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
v.super_block.write().unwrap().replica_placement =
|
||||
crate::storage::super_block::ReplicaPlacement::from_string("001").unwrap();
|
||||
v.set_no_write_or_delete(true);
|
||||
|
||||
v.update_remote_files(|files| files.push(Default::default()))
|
||||
.unwrap();
|
||||
assert!(v.is_no_write_or_delete());
|
||||
|
||||
v.set_writable().unwrap();
|
||||
|
||||
assert!(v.is_read_only());
|
||||
assert!(!v.no_write_flags().0);
|
||||
assert!(v.no_write_flags().1);
|
||||
}
|
||||
|
||||
// Tier-down clears the remote mode and publishes the volume as writable. The
|
||||
// map it booted with is the read-only sorted one, whose put always fails, so
|
||||
// without a rebuild the first write appends to .dat and then cannot be
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -24,7 +24,7 @@ require (
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.7 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
|
||||
@@ -38,8 +38,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
|
||||
@@ -5,6 +5,6 @@ go 1.24.7
|
||||
require github.com/segmentio/kafka-go v0.4.49
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/klauspost/compress v1.18.7 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
)
|
||||
@@ -1,7 +1,7 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
module simple-consumer
|
||||
|
||||
go 1.21
|
||||
go 1.24
|
||||
|
||||
require github.com/segmentio/kafka-go v0.4.47
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.17.0 // indirect
|
||||
github.com/klauspost/compress v1.18.7 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
)
|
||||
@@ -2,8 +2,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
|
||||
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw=
|
||||
github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
||||
@@ -12,41 +12,51 @@ import (
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// The very first write against a fresh cluster must succeed in one shot: the
|
||||
// assign that triggers volume growth waits for it instead of shedding itself
|
||||
// with "volume growth in progress". No client retries — a single request per
|
||||
// assign, like a plain HTTP client.
|
||||
// The very first write against a fresh cluster must succeed: the assign that
|
||||
// triggers volume growth waits for it, and when growth outlasts that wait the
|
||||
// master sheds with a retryable 503/Retry-After. A normal client honors the
|
||||
// hint and retries, so the test does the same within a deadline.
|
||||
func TestColdStartFirstWrite(t *testing.T) {
|
||||
c := StartCluster(t, 3)
|
||||
|
||||
// Each subtest uses its own replication so its volume layout starts cold.
|
||||
t.Run("http_assign_and_write", func(t *testing.T) {
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := client.Get(c.MasterURL() + "/dir/assign?replication=002")
|
||||
if err != nil {
|
||||
t.Fatalf("dir/assign: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("dir/assign returned %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
var assign struct {
|
||||
Fid string `json:"fid"`
|
||||
Url string `json:"url"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &assign); err != nil {
|
||||
t.Fatalf("parse assign response %q: %v", body, err)
|
||||
}
|
||||
if assign.Error != "" || assign.Fid == "" {
|
||||
t.Fatalf("assign failed: %s", body)
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for {
|
||||
resp, err := client.Get(c.MasterURL() + "/dir/assign?replication=002")
|
||||
if err != nil {
|
||||
t.Fatalf("dir/assign: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusServiceUnavailable && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("dir/assign returned %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if err := json.Unmarshal(body, &assign); err != nil {
|
||||
t.Fatalf("parse assign response %q: %v", body, err)
|
||||
}
|
||||
if assign.Error != "" || assign.Fid == "" {
|
||||
t.Fatalf("assign failed: %s", body)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
content := []byte("cold start write")
|
||||
@@ -57,21 +67,28 @@ func TestColdStartFirstWrite(t *testing.T) {
|
||||
|
||||
t.Run("grpc_assign", func(t *testing.T) {
|
||||
grpcDialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
err := pb.WithMasterClient(context.Background(), false, pb.ServerAddress(c.MasterAddress()), grpcDialOption, false,
|
||||
func(client master_pb.SeaweedClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
resp, err := client.Assign(ctx, &master_pb.AssignRequest{Count: 1, Replication: "001"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Error != "" || resp.Fid == "" {
|
||||
return fmt.Errorf("assign failed: %+v", resp)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
var err error
|
||||
for deadline := time.Now().Add(60 * time.Second); ; {
|
||||
err = pb.WithMasterClient(context.Background(), false, pb.ServerAddress(c.MasterAddress()), grpcDialOption, false,
|
||||
func(client master_pb.SeaweedClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
resp, err := client.Assign(ctx, &master_pb.AssignRequest{Count: 1, Replication: "001"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Error != "" || resp.Fid == "" {
|
||||
return fmt.Errorf("assign failed: %+v", resp)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil || status.Code(err) != codes.ResourceExhausted || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("single grpc assign on cold cluster: %v", err)
|
||||
t.Fatalf("grpc assign on cold cluster: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
An integration test for [OLake](https://github.com/datazip-inc/olake) against
|
||||
SeaweedFS's Iceberg REST Catalog, in the same shape as `catalog_clickhouse`.
|
||||
|
||||
## Why OLake, given we already test five engines
|
||||
## Why OLake, given we already test other engines
|
||||
|
||||
Two things here are covered by nothing else in this directory.
|
||||
|
||||
@@ -73,8 +73,9 @@ Nothing in the destination config is SeaweedFS-specific:
|
||||
```
|
||||
|
||||
`catalog_type` is the generic `rest`, auth is the standard OAuth2
|
||||
client-credentials flow, and `s3_path_style` does not even need setting —
|
||||
OLake turns it on by itself whenever `s3_endpoint` is non-empty. As of
|
||||
client-credentials flow, and the test config sets `s3_path_style` explicitly;
|
||||
OLake also defaults to path style whenever `s3_endpoint` is non-empty, so
|
||||
either form works. As of
|
||||
OLake v0.10.1 this works with no change on either side.
|
||||
|
||||
## Running it
|
||||
|
||||
@@ -376,8 +376,8 @@ func (env *TestEnvironment) execSQL(t *testing.T, sqlText string) {
|
||||
// writeOLakeConfigs writes the source and destination configs. The destination
|
||||
// is the point of this test: nothing in it is SeaweedFS-specific. catalog_type
|
||||
// is the generic "rest", auth is the standard OAuth2 client-credentials flow,
|
||||
// and path-style access is not even set here because OLake turns it on by
|
||||
// itself whenever s3_endpoint is non-empty.
|
||||
// and path-style access is set explicitly here; OLake also turns it on by
|
||||
// itself whenever s3_endpoint is non-empty, so either form works.
|
||||
func (env *TestEnvironment) writeOLakeConfigs(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newObjectClient(t *testing.T) *s3.Client {
|
||||
t.Helper()
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(testRegion),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(sharedCluster.s3Endpoint)
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
}
|
||||
|
||||
func apiErrorCode(err error) string {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.ErrorCode()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The external-volume verification object round-trips at a table bucket's
|
||||
// root the way the check uses it: PUT, HEAD, GET, list, DELETE. Any other key
|
||||
// at the root is still refused.
|
||||
func TestTableBucketExternalVolumeVerificationRoundTrip(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
ctx := context.Background()
|
||||
tables := NewS3TablesClient(sharedCluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
bucket := "verify-" + randomString(8)
|
||||
created, err := tables.CreateTableBucket(bucket, nil)
|
||||
require.NoError(t, err)
|
||||
defer tables.DeleteTableBucket(created.ARN)
|
||||
client := newObjectClient(t)
|
||||
|
||||
body := []byte("verification payload!") // 21 bytes
|
||||
key := fmt.Sprintf("verify_%d_%d", time.Now().UnixMilli(), 1081846483)
|
||||
|
||||
// the control: an ordinary key at the root is refused
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String("notes.parquet"), Body: bytes.NewReader(body)})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "AccessDenied", apiErrorCode(err))
|
||||
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(body), ContentLength: aws.Int64(int64(len(body)))})
|
||||
require.NoError(t, err)
|
||||
|
||||
head, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(body)), aws.ToInt64(head.ContentLength))
|
||||
|
||||
got, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(got.Body)
|
||||
got.Body.Close()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, body, data)
|
||||
|
||||
list, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String("verify_")})
|
||||
require.NoError(t, err)
|
||||
var keys []string
|
||||
for _, o := range list.Contents {
|
||||
keys = append(keys, aws.ToString(o.Key))
|
||||
}
|
||||
assert.Equal(t, []string{key}, keys)
|
||||
|
||||
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
require.NoError(t, err)
|
||||
_, err = client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
|
||||
require.Error(t, err, "the verification object must be gone after DELETE")
|
||||
}
|
||||
|
||||
// A namespace can carry a name of the verification key's shape. The key
|
||||
// then names the namespace's directory: a PUT or DELETE through it must not
|
||||
// replace or remove the namespace.
|
||||
func TestTableBucketVerificationKeyLeavesNamespaceAlone(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
ctx := context.Background()
|
||||
tables := NewS3TablesClient(sharedCluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
bucket := "verify-ns-" + randomString(8)
|
||||
created, err := tables.CreateTableBucket(bucket, nil)
|
||||
require.NoError(t, err)
|
||||
defer tables.DeleteTableBucket(created.ARN)
|
||||
client := newObjectClient(t)
|
||||
body := []byte("verification payload!")
|
||||
|
||||
listed := func(name string) bool {
|
||||
resp, err := tables.ListNamespaces(created.ARN, "", "", 0)
|
||||
require.NoError(t, err)
|
||||
for _, ns := range resp.Namespaces {
|
||||
if len(ns.Namespace) == 1 && ns.Namespace[0] == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for i, op := range []string{"PUT", "DELETE"} {
|
||||
t.Run(op, func(t *testing.T) {
|
||||
name := fmt.Sprintf("verify_%d_%d", time.Now().UnixMilli(), 7+i)
|
||||
_, err := tables.CreateNamespace(created.ARN, []string{name})
|
||||
require.NoError(t, err)
|
||||
defer tables.DeleteNamespace(created.ARN, []string{name})
|
||||
require.True(t, listed(name), "setup: the namespace is listed")
|
||||
|
||||
if op == "PUT" {
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(name), Body: bytes.NewReader(body), ContentLength: aws.Int64(int64(len(body)))})
|
||||
assert.Error(t, err, "a PUT over a namespace must be refused")
|
||||
} else {
|
||||
_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(name)})
|
||||
assert.Error(t, err, "a DELETE of a namespace must be refused")
|
||||
}
|
||||
_, err = tables.GetNamespace(created.ARN, []string{name})
|
||||
assert.NoError(t, err, "the namespace survives the %s", op)
|
||||
assert.True(t, listed(name), "the namespace is still listed after the %s", op)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -342,7 +342,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`
|
||||
}
|
||||
<label class="btn btn-outline-primary w-100 text-start p-3" for="s3tablesBucketFormatIceberg">
|
||||
<span class="d-block fw-semibold"><i class="bi bi-snow me-1"></i>Iceberg</span>
|
||||
<span class="d-block small text-muted mt-1">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB.</span>
|
||||
<span class="d-block small text-muted mt-1">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
|
||||
@@ -636,7 +636,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB.</span></label></div><div class=\"col-6\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -384,6 +384,16 @@ func processUpdateEvent(
|
||||
}
|
||||
glog.V(0).Infof("never replicated, uploading %s", remote_storage.FormatLocation(dest))
|
||||
}
|
||||
if proto.Equal(oldDest, dest) && isDroppedLocalCopy(message.NewEntry) {
|
||||
// The update dropped the local copy: remote.uncache, eviction under
|
||||
// storage pressure, or remote.meta.sync pulling a newer remote
|
||||
// version. Each clears the sync stamp, so shouldSendToRemote reads
|
||||
// the entry as dirty when the remote mtime shares its second. The
|
||||
// content is the object at this very key, and uploading the chunkless
|
||||
// entry would replace it with zero bytes.
|
||||
glog.V(2).Infof("skipping update of remote-only entry: %+v", resp)
|
||||
return nil
|
||||
}
|
||||
glog.V(2).Infof("update: %+v", resp)
|
||||
if !proto.Equal(oldDest, dest) {
|
||||
// A renamed entry that holds no local data now (the snapshot was
|
||||
@@ -488,6 +498,15 @@ func isRemoteOnly(entry *filer_pb.Entry) bool {
|
||||
return entry != nil && !filer.HasData(entry) && entry.IsInRemoteOnly()
|
||||
}
|
||||
|
||||
// isDroppedLocalCopy reports a remote-only entry whose size is still the
|
||||
// remote object's: its local copy was dropped and the object describes it.
|
||||
// Uncache and eviction keep the size, and remote.meta.sync sets it from the
|
||||
// object. A truncate drops the chunks too but changes the size, and the
|
||||
// truncated content has to reach the remote.
|
||||
func isDroppedLocalCopy(entry *filer_pb.Entry) bool {
|
||||
return isRemoteOnly(entry) && entry.GetAttributes().GetFileSize() == uint64(entry.RemoteEntry.RemoteSize)
|
||||
}
|
||||
|
||||
// completeRemoteOnlyRename finishes a rename whose content exists only on the
|
||||
// remote. When the destination object already holds what the entry's stamp
|
||||
// describes (the rename ran before, its offset was not persisted, and
|
||||
|
||||
@@ -1722,6 +1722,206 @@ func TestMetadataOnlyUpdateRestampsSyncTs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The droppedCopy fixtures are a small synced file under a mount: a Delta log
|
||||
// entry whose remote object was written in the second of its mtime, as a quick
|
||||
// sync of a small file is.
|
||||
const droppedCopyMtime = 1786096669
|
||||
|
||||
var (
|
||||
droppedCopyDir = "/buckets/gold/sales/_delta_log"
|
||||
droppedCopyName = "00000000000000000007.json"
|
||||
droppedCopyMount = &remote_pb.RemoteStorageLocation{Name: "s3", Bucket: "gold", Path: "/"}
|
||||
droppedCopyDest = &remote_pb.RemoteStorageLocation{Name: "s3", Bucket: "gold", Path: "/sales/_delta_log/" + droppedCopyName}
|
||||
droppedCopySynced = &filer_pb.RemoteEntry{StorageName: "s3", RemoteETag: `"abc"`, RemoteSize: 1024, RemoteMtime: droppedCopyMtime, LastLocalSyncTsNs: droppedCopyMtime*1e9 + 500}
|
||||
)
|
||||
|
||||
// droppedCopyStamp is the synced stamp with LastLocalSyncTsNs cleared, as
|
||||
// remote.uncache and the eviction leave it.
|
||||
func droppedCopyStamp() *filer_pb.RemoteEntry {
|
||||
stamp := proto.Clone(droppedCopySynced).(*filer_pb.RemoteEntry)
|
||||
stamp.LastLocalSyncTsNs = 0
|
||||
return stamp
|
||||
}
|
||||
|
||||
// droppedCopyEntry is the fixture file with the given stamp and chunks, sized
|
||||
// as its remote object.
|
||||
func droppedCopyEntry(stamp *filer_pb.RemoteEntry, chunks ...*filer_pb.FileChunk) *filer_pb.Entry {
|
||||
entry := entryWith(droppedCopyName, stamp, chunks...)
|
||||
entry.Attributes.FileSize = uint64(droppedCopySynced.RemoteSize)
|
||||
return entry
|
||||
}
|
||||
|
||||
func droppedCopyUpdate(oldEntry, newEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: droppedCopyDir,
|
||||
EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewParentPath: droppedCopyDir, NewEntry: newEntry},
|
||||
}
|
||||
}
|
||||
|
||||
// TestDroppedLocalCopyIsNotUploaded: remote.uncache and the filer's eviction
|
||||
// under storage pressure drop a synced entry's chunks and clear its sync
|
||||
// stamp; remote.meta.sync does the same when it pulls a newer remote version.
|
||||
// With the remote mtime in the entry's second, shouldSendToRemote falls back
|
||||
// to the cleared stamp and reads the entry as dirty. The object at the same
|
||||
// key is the entry's only content, so uploading the chunkless entry would
|
||||
// replace it with a zero-byte object. The update is skipped, and the stamp
|
||||
// stays cleared so remote.cache and eviction still see the entry as uncached.
|
||||
func TestDroppedLocalCopyIsNotUploaded(t *testing.T) {
|
||||
// the eviction's entry, built the way uncacheRemoteEntry builds it, and
|
||||
// the event the way the filer's CreateEntry emits it
|
||||
cached := &filer.Entry{
|
||||
FullPath: util.NewFullPath(droppedCopyDir, droppedCopyName),
|
||||
Attr: filer.Attr{Mtime: time.Unix(droppedCopyMtime, 0), Mode: 0660, FileSize: 1024},
|
||||
Chunks: []*filer_pb.FileChunk{chunk("3,01", "e1")},
|
||||
Remote: droppedCopySynced,
|
||||
}
|
||||
evicted := cached.ShallowClone()
|
||||
evicted.Chunks = nil
|
||||
evicted.Remote = proto.Clone(cached.Remote).(*filer_pb.RemoteEntry)
|
||||
evicted.Remote.LastLocalSyncTsNs = 0
|
||||
|
||||
// remote.meta.sync found a newer object: its mtime becomes the entry's
|
||||
newer := &filer_pb.RemoteEntry{StorageName: "s3", RemoteETag: `"def"`, RemoteSize: 4096, RemoteMtime: droppedCopyMtime + 60}
|
||||
pulled := &filer_pb.Entry{Name: droppedCopyName, Attributes: &filer_pb.FuseAttributes{Mtime: droppedCopyMtime + 60, FileSize: 4096}, RemoteEntry: newer}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
oldEntry *filer_pb.Entry
|
||||
newEntry *filer_pb.Entry
|
||||
}{
|
||||
{name: "remote.uncache", oldEntry: droppedCopyEntry(droppedCopySynced, chunk("3,01", "e1")), newEntry: droppedCopyEntry(droppedCopyStamp())},
|
||||
{name: "eviction under storage pressure", oldEntry: cached.ToProtoEntry(), newEntry: evicted.ToProtoEntry()},
|
||||
{name: "remote.meta.sync pulls a newer version", oldEntry: entryWith(droppedCopyName, droppedCopySynced, chunk("3,01", "e1")), newEntry: pulled},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp := droppedCopyUpdate(tt.oldEntry, tt.newEntry)
|
||||
if !shouldSendToRemote(tt.newEntry) {
|
||||
t.Fatal("precondition: the cleared stamp in the remote mtime's second reads as dirty, the bug's trigger")
|
||||
}
|
||||
if isMetadataOnlyUpdate(resp.Directory, resp.EventNotification) {
|
||||
t.Fatal("precondition: the dropped chunks make the update a content change")
|
||||
}
|
||||
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: tt.newEntry}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.writes) != 0 {
|
||||
t.Errorf("writes = %+v, want none: the remote object is the entry's content and an upload would zero it", remote.writes)
|
||||
}
|
||||
if len(remote.deletes) != 0 || len(remote.metadataUpdates) != 0 {
|
||||
t.Errorf("deletes = %+v, metadata updates = %+v, want the remote object left alone", remote.deletes, remote.metadataUpdates)
|
||||
}
|
||||
if filerClient.updates != 0 {
|
||||
t.Errorf("stamped the entry %d times, want 0: a stamp would mark the uncached entry as cached", filerClient.updates)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a metadata edit of a remote-only entry still reaches the remote", func(t *testing.T) {
|
||||
tagged := droppedCopyEntry(droppedCopyStamp())
|
||||
tagged.Extended = map[string][]byte{"X-Amz-Tagging-tier": []byte("gold")}
|
||||
resp := droppedCopyUpdate(droppedCopyEntry(droppedCopyStamp()), tagged)
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: tagged}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.metadataUpdates) != 1 || len(remote.writes) != 0 {
|
||||
t.Errorf("metadata updates = %+v, writes = %+v, want one metadata update and no upload", remote.metadataUpdates, remote.writes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestSameSecondRewriteIsStillUploaded: a rewrite that lands in the remote
|
||||
// mtime's second holds local data, so the skip for dropped local copies does
|
||||
// not apply and it is uploaded and restamped (#11715).
|
||||
func TestSameSecondRewriteIsStillUploaded(t *testing.T) {
|
||||
stale := proto.Clone(droppedCopySynced).(*filer_pb.RemoteEntry)
|
||||
stale.LastLocalSyncTsNs = droppedCopyMtime*1e9 + 100
|
||||
rewrite := func(stamp *filer_pb.RemoteEntry) *filer_pb.Entry {
|
||||
entry := entryWith(droppedCopyName, stamp, chunk("4,02", "e2"))
|
||||
entry.Attributes.MtimeNs = 337
|
||||
return entry
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
oldEntry *filer_pb.Entry
|
||||
newEntry *filer_pb.Entry
|
||||
}{
|
||||
// Filer.UpdateEntry clears the stamp of a content change
|
||||
{name: "synced entry, stamp cleared by the filer", oldEntry: entryWith(droppedCopyName, droppedCopySynced, chunk("3,01", "e1")), newEntry: rewrite(droppedCopyStamp())},
|
||||
{name: "synced entry, stamp older than the rewrite", oldEntry: entryWith(droppedCopyName, stale, chunk("3,01", "e1")), newEntry: rewrite(stale)},
|
||||
{name: "uncached entry", oldEntry: entryWith(droppedCopyName, droppedCopyStamp()), newEntry: rewrite(droppedCopyStamp())},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.newEntry.Attributes.Mtime != tt.newEntry.RemoteEntry.RemoteMtime {
|
||||
t.Fatal("precondition: the rewrite lands in the remote mtime's second")
|
||||
}
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: tt.newEntry}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, droppedCopyUpdate(tt.oldEntry, tt.newEntry)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], droppedCopyDest) {
|
||||
t.Fatalf("writes = %+v, want the rewrite uploaded to %s", remote.writes, remote_storage.FormatLocation(droppedCopyDest))
|
||||
}
|
||||
if !slices.Equal(remote.writtenChunks[0], []string{"4,02"}) {
|
||||
t.Errorf("uploaded chunks %v, want the rewrite's [4,02]", remote.writtenChunks[0])
|
||||
}
|
||||
if filerClient.updates != 1 {
|
||||
t.Errorf("stamped the entry %d times, want 1 after the upload", filerClient.updates)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTruncateToZeroIsUploaded: a FUSE truncate of a cached file to zero
|
||||
// (truncateEntry, or SetAttr with size 0) drops the chunks and keeps the
|
||||
// sized RemoteEntry, so the entry looks remote-only. Its size is no longer
|
||||
// the remote object's, and the empty file has to replace the object.
|
||||
func TestTruncateToZeroIsUploaded(t *testing.T) {
|
||||
truncated := func(mtime int64, mtimeNs int32) *filer_pb.Entry {
|
||||
// the filer clears the stamp of a content change
|
||||
entry := entryWith(droppedCopyName, droppedCopyStamp())
|
||||
entry.Attributes.Mtime, entry.Attributes.MtimeNs = mtime, mtimeNs
|
||||
return entry
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
newEntry *filer_pb.Entry
|
||||
}{
|
||||
{name: "in a later second", newEntry: truncated(droppedCopyMtime+5, 0)},
|
||||
{name: "in the remote mtime's second", newEntry: truncated(droppedCopyMtime, 337)},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if !isRemoteOnly(tt.newEntry) {
|
||||
t.Fatal("precondition: the truncated entry looks remote-only")
|
||||
}
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: tt.newEntry}
|
||||
resp := droppedCopyUpdate(droppedCopyEntry(droppedCopySynced, chunk("3,01", "e1")), tt.newEntry)
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, "/buckets/gold", droppedCopyMount, resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], droppedCopyDest) {
|
||||
t.Fatalf("writes = %+v, want the empty file uploaded to %s", remote.writes, remote_storage.FormatLocation(droppedCopyDest))
|
||||
}
|
||||
if len(remote.writtenChunks[0]) != 0 {
|
||||
t.Errorf("uploaded chunks %v, want none: a zero-byte object", remote.writtenChunks[0])
|
||||
}
|
||||
if filerClient.updates != 1 {
|
||||
t.Errorf("stamped the entry %d times, want 1 after the upload", filerClient.updates)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameKeepsAReusedOldKey(t *testing.T) {
|
||||
// A was renamed to B, then D to A before the sync got to the first
|
||||
// rename: the object at A is now D's, and the only copy of it
|
||||
|
||||
+4
-4
@@ -762,8 +762,8 @@ type icebergCredentialVendor struct {
|
||||
server *s3api.S3ApiServer
|
||||
}
|
||||
|
||||
func (v icebergCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*iceberg.VendedCredentials, error) {
|
||||
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix)
|
||||
func (v icebergCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*iceberg.VendedCredentials, error) {
|
||||
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
|
||||
if err != nil || credentials == nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -781,8 +781,8 @@ type lanceCredentialVendor struct {
|
||||
server *s3api.S3ApiServer
|
||||
}
|
||||
|
||||
func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*lance.VendedCredentials, error) {
|
||||
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix)
|
||||
func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*lance.VendedCredentials, error) {
|
||||
credentials, err := v.server.VendTableCredentials(ctx, principal, bucket, prefix, readOnly)
|
||||
if err != nil || credentials == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -28,6 +28,10 @@ region = "us-east-2"
|
||||
bucket = "your_bucket_name" # an existing bucket
|
||||
endpoint = ""
|
||||
storage_class = "STANDARD_IA"
|
||||
# optional HTTP client tuning for S3-compatible services:
|
||||
# disable_http2 = false # true keeps the client on HTTP/1.1, one connection per concurrent request
|
||||
# max_connections = 0 # cap on connections to the endpoint, 0 = no cap
|
||||
# idle_connection_timeout_seconds = 0 # close idle connections sooner, 0 = Go default (90s)
|
||||
|
||||
# create this number of logical volumes if no more writable volumes
|
||||
# count_x means how many copies of data.
|
||||
|
||||
+33
-19
@@ -20,6 +20,10 @@ var (
|
||||
upload UploadOptions
|
||||
)
|
||||
|
||||
// uploadBatchSize is how many files a -dir upload submits per batch: one
|
||||
// assign RPC fetches all their file ids, and the batch uploads in parallel.
|
||||
const uploadBatchSize = 32
|
||||
|
||||
type UploadOptions struct {
|
||||
master *string
|
||||
dir *string
|
||||
@@ -85,6 +89,7 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
if *upload.dir == "" {
|
||||
return false
|
||||
}
|
||||
var paths []string
|
||||
err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
|
||||
if err == nil {
|
||||
if !info.IsDir() {
|
||||
@@ -93,23 +98,7 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
parts, e := operation.NewFileParts([]string{path})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
|
||||
Replication: *upload.replication,
|
||||
Collection: *upload.collection,
|
||||
DataCenter: *upload.dataCenter,
|
||||
Ttl: *upload.ttl,
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
paths = append(paths, path)
|
||||
}
|
||||
} else {
|
||||
fmt.Println(err)
|
||||
@@ -120,6 +109,31 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
fmt.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
hadErr := false
|
||||
for batchStart := 0; batchStart < len(paths); batchStart += uploadBatchSize {
|
||||
batchEnd := min(batchStart+uploadBatchSize, len(paths))
|
||||
parts, e := operation.NewFileParts(paths[batchStart:batchEnd])
|
||||
if e != nil {
|
||||
fmt.Println(e.Error())
|
||||
hadErr = true
|
||||
continue
|
||||
}
|
||||
results, e := operation.SubmitFiles(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, operation.StoragePreference{
|
||||
Replication: *upload.replication,
|
||||
Collection: *upload.collection,
|
||||
DataCenter: *upload.dataCenter,
|
||||
Ttl: *upload.ttl,
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if e != nil {
|
||||
fmt.Println(e.Error())
|
||||
hadErr = true
|
||||
}
|
||||
}
|
||||
return !hadErr
|
||||
} else {
|
||||
parts, e := operation.NewFileParts(args)
|
||||
if e != nil {
|
||||
@@ -134,12 +148,12 @@ func runUpload(cmd *Command, args []string) bool {
|
||||
DiskType: *upload.diskType,
|
||||
MaxMB: *upload.maxMB,
|
||||
}, *upload.usePublicUrl)
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
bytes, _ := json.Marshal(results)
|
||||
fmt.Println(string(bytes))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -120,13 +120,20 @@ func (store *IamGrpcStore) CreateAccessKey(ctx context.Context, username string,
|
||||
}
|
||||
|
||||
func (store *IamGrpcStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
|
||||
return store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
|
||||
err := store.withIamClient(ctx, func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
|
||||
_, err := client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
|
||||
Username: username,
|
||||
AccessKey: accessKey,
|
||||
})
|
||||
return err
|
||||
})
|
||||
// The filer-side handler returns gRPC NotFound when the user or the key is
|
||||
// absent; translate back to the sentinel so callers can use
|
||||
// errors.Is(err, credential.ErrAccessKeyNotFound) uniformly across stores.
|
||||
if err != nil && status.Code(err) == codes.NotFound {
|
||||
return credential.ErrAccessKeyNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// AttachUserPolicy attaches a managed policy to a user by policy name
|
||||
|
||||
@@ -2,6 +2,7 @@ package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
@@ -106,3 +107,50 @@ func TestIamGrpcStore_AdminBearerToken(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestIamGrpcStore_DeleteAccessKeyNotFound pins the not-found translation:
|
||||
// the filer reports a missing user or key as gRPC NotFound, and the store must
|
||||
// surface it as credential.ErrAccessKeyNotFound so repeat deletes stay a no-op
|
||||
// for IAM dispatchers that tolerate the sentinel.
|
||||
func TestIamGrpcStore_DeleteAccessKeyNotFound(t *testing.T) {
|
||||
const goodKey = "iam-admin-itest-key"
|
||||
|
||||
cm, err := credential.NewCredentialManager(credential.StoreTypeMemory, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("NewCredentialManager: %v", err)
|
||||
}
|
||||
defer cm.Shutdown()
|
||||
|
||||
iamSrv := weed_server.NewIamGrpcServer(cm, security.SigningKey(goodKey))
|
||||
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcSrv := grpc.NewServer()
|
||||
iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcSrv, iamSrv)
|
||||
go func() { _ = grpcSrv.Serve(lis) }()
|
||||
t.Cleanup(grpcSrv.Stop)
|
||||
|
||||
_, portStr, err := net.SplitHostPort(lis.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("split listener addr: %v", err)
|
||||
}
|
||||
serverAddr := pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", portStr))
|
||||
|
||||
store := &IamGrpcStore{}
|
||||
store.SetFilerAddressFunc(func() pb.ServerAddress { return serverAddr }, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
store.SetAdminSigning(security.SigningKey(goodKey), 0)
|
||||
|
||||
ctx := context.Background()
|
||||
if err := store.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil {
|
||||
t.Fatalf("CreateUser: %v", err)
|
||||
}
|
||||
|
||||
if err := store.DeleteAccessKey(ctx, "alice", "AKIAMISSING0000000001"); !errors.Is(err, credential.ErrAccessKeyNotFound) {
|
||||
t.Fatalf("DeleteAccessKey with missing key: got %v, want ErrAccessKeyNotFound", err)
|
||||
}
|
||||
if err := store.DeleteAccessKey(ctx, "ghost", "AKIAMISSING0000000001"); !errors.Is(err, credential.ErrAccessKeyNotFound) {
|
||||
t.Fatalf("DeleteAccessKey with missing user: got %v, want ErrAccessKeyNotFound", err)
|
||||
}
|
||||
}
|
||||
@@ -142,6 +142,14 @@ func (store *MemoryStore) UpdateUser(ctx context.Context, username string, ident
|
||||
return credential.ErrUserNotFound
|
||||
}
|
||||
|
||||
if identity.Name != username {
|
||||
// Rename: the destination name must not already be taken, matching
|
||||
// RenameUser's collision check.
|
||||
if _, clash := store.users[identity.Name]; clash {
|
||||
return credential.ErrUserAlreadyExists
|
||||
}
|
||||
}
|
||||
|
||||
// Remove old access keys from index
|
||||
for _, cred := range existingUser.Credentials {
|
||||
delete(store.accessKeys, cred.AccessKey)
|
||||
@@ -156,11 +164,15 @@ func (store *MemoryStore) UpdateUser(ctx context.Context, username string, ident
|
||||
|
||||
// Deep copy to avoid mutation issues
|
||||
identityCopy := store.deepCopyIdentity(identity)
|
||||
store.users[username] = identityCopy
|
||||
if identityCopy.Name != username {
|
||||
// Rename: re-key the record, the way filer_etc does (delete old, save new).
|
||||
delete(store.users, username)
|
||||
}
|
||||
store.users[identityCopy.Name] = identityCopy
|
||||
|
||||
// Re-index access keys
|
||||
for _, cred := range identity.Credentials {
|
||||
store.accessKeys[cred.AccessKey] = username
|
||||
store.accessKeys[cred.AccessKey] = identityCopy.Name
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -378,3 +378,82 @@ func TestMemoryStoreServiceAccountByAccessKey(t *testing.T) {
|
||||
t.Errorf("Expected ErrAccessKeyNotFound after delete, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMemoryStoreUpdateUserRename: UpdateUser with an identity whose Name
|
||||
// differs from the key must re-key the record, the way filer_etc does
|
||||
// (delete old, save new). The IAM rename path relies on this.
|
||||
func TestMemoryStoreUpdateUserRename(t *testing.T) {
|
||||
store := &MemoryStore{}
|
||||
if err := store.Initialize(util.GetViper(), "credential."); err != nil {
|
||||
t.Fatalf("Failed to initialize store: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
err := store.CreateUser(ctx, &iam_pb.Identity{
|
||||
Name: "oldname",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "access123", SecretKey: "secret123"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create user: %v", err)
|
||||
}
|
||||
|
||||
err = store.UpdateUser(ctx, "oldname", &iam_pb.Identity{
|
||||
Name: "newname",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "access123", SecretKey: "secret123"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to rename user via UpdateUser: %v", err)
|
||||
}
|
||||
|
||||
if _, err := store.GetUser(ctx, "oldname"); err != credential.ErrUserNotFound {
|
||||
t.Fatalf("Expected ErrUserNotFound for old name, got %v", err)
|
||||
}
|
||||
renamed, err := store.GetUser(ctx, "newname")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get renamed user: %v", err)
|
||||
}
|
||||
if renamed.Name != "newname" {
|
||||
t.Errorf("Expected 'newname', got '%s'", renamed.Name)
|
||||
}
|
||||
byKey, err := store.GetUserByAccessKey(ctx, "access123")
|
||||
if err != nil {
|
||||
t.Fatalf("Access key must resolve to the renamed user: %v", err)
|
||||
}
|
||||
if byKey.Name != "newname" {
|
||||
t.Errorf("Expected access key owner 'newname', got '%s'", byKey.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMemoryStoreUpdateUserRenameCollision: a rename whose destination name is
|
||||
// taken must fail rather than overwrite the existing user.
|
||||
func TestMemoryStoreUpdateUserRenameCollision(t *testing.T) {
|
||||
store := &MemoryStore{}
|
||||
if err := store.Initialize(util.GetViper(), "credential."); err != nil {
|
||||
t.Fatalf("Failed to initialize store: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
if err := store.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil {
|
||||
t.Fatalf("Failed to create alice: %v", err)
|
||||
}
|
||||
if err := store.CreateUser(ctx, &iam_pb.Identity{
|
||||
Name: "bob",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "bobkey123", SecretKey: "secret123"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed to create bob: %v", err)
|
||||
}
|
||||
|
||||
err := store.UpdateUser(ctx, "alice", &iam_pb.Identity{Name: "bob"})
|
||||
if err != credential.ErrUserAlreadyExists {
|
||||
t.Fatalf("Expected ErrUserAlreadyExists, got %v", err)
|
||||
}
|
||||
|
||||
// Both users must be intact.
|
||||
if _, err := store.GetUser(ctx, "alice"); err != nil {
|
||||
t.Fatalf("alice must still exist: %v", err)
|
||||
}
|
||||
bob, err := store.GetUserByAccessKey(ctx, "bobkey123")
|
||||
if err != nil || bob.Name != "bob" {
|
||||
t.Fatalf("bob's access key must still resolve to bob: %v", err)
|
||||
}
|
||||
}
|
||||
+9
-7
@@ -104,13 +104,15 @@ var baseS3ActionMap = map[string]string{
|
||||
"DeleteObjectVersion": s3_constants.ACTION_WRITE,
|
||||
"GetObjectVersion": s3_constants.ACTION_READ,
|
||||
// Tagging operations
|
||||
"GetObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"GetObjectVersionTagging": s3_constants.ACTION_TAGGING,
|
||||
"PutObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"DeleteObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"GetBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
"PutBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
"DeleteBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
"GetObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"GetObjectVersionTagging": s3_constants.ACTION_TAGGING,
|
||||
"PutObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"PutObjectVersionTagging": s3_constants.ACTION_TAGGING,
|
||||
"DeleteObjectTagging": s3_constants.ACTION_TAGGING,
|
||||
"DeleteObjectVersionTagging": s3_constants.ACTION_TAGGING,
|
||||
"GetBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
"PutBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
"DeleteBucketTagging": s3_constants.ACTION_TAGGING,
|
||||
// ACL operations
|
||||
"PutObjectAcl": s3_constants.ACTION_WRITE_ACP,
|
||||
"GetObjectAcl": s3_constants.ACTION_READ_ACP,
|
||||
|
||||
@@ -2,10 +2,16 @@ package iamapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -141,3 +147,69 @@ func TestIamManagementAllowedForAdmin(t *testing.T) {
|
||||
"", "Action=CreateUser&UserName=new-user&Version=2010-05-08")
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
// The implicit UserName must come from the identity authentication verified,
|
||||
// never from re-parsing the Authorization header: a request can carry presigned
|
||||
// credentials for one user alongside a junk SigV4 header naming another.
|
||||
func TestImplicitUserNameUsesVerifiedIdentity(t *testing.T) {
|
||||
server := newAuthzTestServer(t)
|
||||
expires := strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10)
|
||||
sts := "POST\n\napplication/x-www-form-urlencoded\n" + expires + "\n/"
|
||||
mac := hmac.New(sha1.New, []byte("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))
|
||||
mac.Write([]byte(sts))
|
||||
sig := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
q := url.Values{"AWSAccessKeyId": {"AKIAIOSFODNN7EXAMPLE"}, "Expires": {expires}, "Signature": {sig}}
|
||||
|
||||
post := func(t *testing.T, body string) (int, string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodPost, server.URL+"/?"+q.Encode(), strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIATESTFAKEADMIN0001/20261021/us-east-1/iam/aws4_request, SignedHeaders=host, Signature=00")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
t.Logf("%d %s", resp.StatusCode, b)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
for _, action := range []string{"ListAccessKeys", "CreateAccessKey", "GetUser"} {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
code, body := post(t, "Action="+action+"&Version=2010-05-08")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.NotContains(t, body, "iam_admin")
|
||||
assert.Contains(t, body, "power_user")
|
||||
})
|
||||
}
|
||||
|
||||
// ListUserPolicies is not self-service: a caller without IAM rights is
|
||||
// denied whichever name the implicit resolution lands on.
|
||||
t.Run("ListUserPolicies", func(t *testing.T) {
|
||||
code, body := post(t, "Action=ListUserPolicies&Version=2010-05-08")
|
||||
assert.Equal(t, http.StatusForbidden, code)
|
||||
assert.NotContains(t, body, "iam_admin")
|
||||
})
|
||||
|
||||
// A write aimed at the header-named admin's key resolves to the verified
|
||||
// caller instead: the key is not power_user's, so it is not found.
|
||||
t.Run("UpdateAccessKey", func(t *testing.T) {
|
||||
code, body := post(t, "Action=UpdateAccessKey&AccessKeyId=AKIATESTFAKEADMIN0001&Status=Inactive&Version=2010-05-08")
|
||||
assert.Equal(t, http.StatusNotFound, code)
|
||||
assert.Contains(t, body, "NoSuchEntity")
|
||||
})
|
||||
|
||||
t.Run("DeleteAccessKey", func(t *testing.T) {
|
||||
code, _ := post(t, "Action=DeleteAccessKey&AccessKeyId=AKIATESTFAKEADMIN0001&Version=2010-05-08")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
})
|
||||
|
||||
// The admin key the junk header named is untouched: still present and active.
|
||||
resp := postSignedIamAction(t, server.URL, "AKIATESTFAKEADMIN0001", "testAdminSecretFake0000000000000000000000",
|
||||
"", "Action=ListAccessKeys&Version=2010-05-08")
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(body), "AKIATESTFAKEADMIN0001")
|
||||
assert.Contains(t, string(body), "<Status>Active</Status>")
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package iamapi
|
||||
// Common IAM types and helpers are imported from the shared weed/iam package.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -14,12 +15,14 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
iamlib "github.com/seaweedfs/seaweedfs/weed/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/request_id"
|
||||
)
|
||||
@@ -1263,53 +1266,128 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
|
||||
return resp
|
||||
}
|
||||
|
||||
// handleImplicitUsername adds username who signs the request to values if 'username' is not specified.
|
||||
// According to AWS documentation: "If you do not specify a user name, IAM determines the user name
|
||||
// implicitly based on the Amazon Web Services access key ID signing the request."
|
||||
// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding
|
||||
// identity name in the credential store.
|
||||
// handleImplicitUsername adds the name of the authenticated caller to values
|
||||
// when 'UserName' is not specified, per AWS: "If you do not specify a user name,
|
||||
// IAM determines the user name implicitly based on the Amazon Web Services
|
||||
// access key ID signing the request." The name comes from the identity the
|
||||
// authentication middleware verified, which the request context carries; the
|
||||
// Authorization header itself is attacker-controlled and is never re-parsed.
|
||||
func (iama *IamApiServer) handleImplicitUsername(r *http.Request, values url.Values) {
|
||||
if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" {
|
||||
if values.Get("UserName") != "" {
|
||||
return
|
||||
}
|
||||
// Log presence of auth header without exposing sensitive signature material
|
||||
glog.V(4).Infof("Authorization header present, extracting access key")
|
||||
// Parse AWS SigV4 Authorization header format:
|
||||
// "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..."
|
||||
s := strings.Split(r.Header["Authorization"][0], "Credential=")
|
||||
if len(s) < 2 {
|
||||
return
|
||||
if name := s3_constants.GetIdentityNameFromContext(r); name != "" {
|
||||
values.Set("UserName", name)
|
||||
}
|
||||
s = strings.Split(s[1], ",")
|
||||
if len(s) < 1 {
|
||||
return
|
||||
}
|
||||
s = strings.Split(s[0], "/")
|
||||
if len(s) < 1 {
|
||||
return
|
||||
}
|
||||
// s[0] is the AccessKeyId
|
||||
accessKeyId := s[0]
|
||||
if accessKeyId == "" {
|
||||
return
|
||||
}
|
||||
// Nil-guard: ensure iam is initialized before lookup
|
||||
}
|
||||
|
||||
// targetedCredentialManager returns the credential manager through which a change to the named
|
||||
// identity is written on its own instead of via SaveConfiguration, or nil when this server has
|
||||
// none, or the identity is static: static identities live outside the store and keep the
|
||||
// full-save path.
|
||||
func (iama *IamApiServer) targetedCredentialManager(userName string) *credential.CredentialManager {
|
||||
if iama.iam == nil {
|
||||
glog.V(4).Infof("IAM not initialized, cannot look up access key")
|
||||
return
|
||||
return nil
|
||||
}
|
||||
// Look up the identity by access key to get the username
|
||||
identity, _, found := iama.iam.LookupByAccessKey(accessKeyId)
|
||||
if !found {
|
||||
// Mask access key in logs - show only first 4 chars
|
||||
maskedKey := accessKeyId
|
||||
if len(accessKeyId) > 4 {
|
||||
maskedKey = accessKeyId[:4] + "***"
|
||||
cm := iama.iam.GetCredentialManager()
|
||||
if cm == nil || cm.IsStaticIdentity(userName) {
|
||||
return nil
|
||||
}
|
||||
return cm
|
||||
}
|
||||
|
||||
// findIdentityByName returns the identity named userName in s3cfg, or nil.
|
||||
func findIdentityByName(s3cfg *iam_pb.S3ApiConfiguration, userName string) *iam_pb.Identity {
|
||||
for _, ident := range s3cfg.Identities {
|
||||
if ident.Name == userName {
|
||||
return ident
|
||||
}
|
||||
glog.V(4).Infof("Access key %s not found in credential store", maskedKey)
|
||||
return
|
||||
}
|
||||
values.Set("UserName", identity.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// groupsWithMember returns the groups in s3cfg that list userName as a member. Taken before a
|
||||
// handler renames or removes the user, it names the group records that handler will change.
|
||||
func groupsWithMember(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.Group {
|
||||
var groups []*iam_pb.Group
|
||||
for _, g := range s3cfg.Groups {
|
||||
for _, m := range g.Members {
|
||||
if m == userName {
|
||||
groups = append(groups, g)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
// serviceAccountsOf returns the service accounts in s3cfg whose parent is userName.
|
||||
func serviceAccountsOf(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.ServiceAccount {
|
||||
var accounts []*iam_pb.ServiceAccount
|
||||
for _, sa := range s3cfg.ServiceAccounts {
|
||||
if sa.ParentUser == userName {
|
||||
accounts = append(accounts, sa)
|
||||
}
|
||||
}
|
||||
return accounts
|
||||
}
|
||||
|
||||
// persistIdentity writes the one identity a handler changed in s3cfg through the credential
|
||||
// manager. The store writes that identity's record and propagates it alone; the other
|
||||
// identities are not rewritten.
|
||||
func persistIdentity(ctx context.Context, cm *credential.CredentialManager, s3cfg *iam_pb.S3ApiConfiguration, userName string) error {
|
||||
ident := findIdentityByName(s3cfg, userName)
|
||||
if ident == nil {
|
||||
return credential.ErrUserNotFound
|
||||
}
|
||||
return cm.UpdateUser(ctx, userName, ident)
|
||||
}
|
||||
|
||||
// persistRenamedIdentity writes a user renamed from oldName to newName by UpdateUser through the
|
||||
// credential manager: the identity, the groups that listed the old name and the service accounts
|
||||
// it parented. UpdateUser under the old name moves the record in stores that treat a changed Name
|
||||
// as a rename (filer_etc, memory), and the propagating store propagates the new identity and the
|
||||
// old name's removal; UpdateUser already moved the inline policies through PutPolicies. A store
|
||||
// that did not move the record still holds the user under the old name; the caller is told so
|
||||
// (false) and keeps the full save, which is what renames such a store today.
|
||||
func persistRenamedIdentity(ctx context.Context, cm *credential.CredentialManager, s3cfg *iam_pb.S3ApiConfiguration, oldName, newName string, groups []*iam_pb.Group, accounts []*iam_pb.ServiceAccount) (bool, error) {
|
||||
ident := findIdentityByName(s3cfg, newName)
|
||||
if ident == nil {
|
||||
return false, credential.ErrUserNotFound
|
||||
}
|
||||
if err := cm.UpdateUser(ctx, oldName, ident); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := cm.GetUser(ctx, newName); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
for _, g := range groups {
|
||||
if err := cm.UpdateGroup(ctx, g); err != nil {
|
||||
return false, fmt.Errorf("update group %s: %w", g.Name, err)
|
||||
}
|
||||
}
|
||||
for _, sa := range accounts {
|
||||
if err := cm.UpdateServiceAccount(ctx, sa.Id, sa); err != nil {
|
||||
return false, fmt.Errorf("update service account %s: %w", sa.Id, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// deleteIdentity removes a user deleted by DeleteUser through the credential manager: its record
|
||||
// (the store propagates the one removal) and the groups that listed it, whose membership
|
||||
// DeleteUser already changed in s3cfg. DeleteUser already removed the inline policies through
|
||||
// PutPolicies.
|
||||
func deleteIdentity(ctx context.Context, cm *credential.CredentialManager, userName string, groups []*iam_pb.Group) error {
|
||||
if err := cm.DeleteUser(ctx, userName); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if err := cm.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("update group %s: %w", g.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1333,6 +1411,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
glog.V(4).Infof("DoActions: %+v", iamlib.RedactSensitiveFormValues(values))
|
||||
var response iamlib.RequestIDSetter
|
||||
changed := true
|
||||
// persisted is set by the actions that wrote one identity through the credential manager
|
||||
// instead of SaveConfiguration; the in-memory identity maps are reloaded after them too.
|
||||
persisted := false
|
||||
switch r.Form.Get("Action") {
|
||||
case "ListUsers":
|
||||
response = iama.ListUsers(s3cfg, values)
|
||||
@@ -1356,6 +1437,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
case "GetUser":
|
||||
iama.handleImplicitUsername(r, values)
|
||||
userName := values.Get("UserName")
|
||||
var err *IamError
|
||||
response, err = iama.GetUser(s3cfg, userName)
|
||||
@@ -1365,40 +1447,107 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
changed = false
|
||||
case "UpdateUser":
|
||||
userName, newUserName := values.Get("UserName"), values.Get("NewUserName")
|
||||
// Taken before the rename: the records the handler changes besides the identity.
|
||||
groups := groupsWithMember(s3cfg, userName)
|
||||
accounts := serviceAccountsOf(s3cfg, userName)
|
||||
var err *IamError
|
||||
response, err = iama.UpdateUser(s3cfg, values)
|
||||
if err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
// Write the renamed identity, its groups and service accounts through the credential
|
||||
// manager instead of rewriting every identity via SaveConfiguration. A request that
|
||||
// renames nothing changed nothing, so there is nothing to write either way.
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
renamed := true
|
||||
if newUserName != "" && newUserName != userName {
|
||||
var err error
|
||||
renamed, err = persistRenamedIdentity(r.Context(), cm, s3cfg, userName, newUserName, groups, accounts)
|
||||
if err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
}
|
||||
changed, persisted = !renamed, renamed
|
||||
}
|
||||
case "DeleteUser":
|
||||
userName := values.Get("UserName")
|
||||
groups := groupsWithMember(s3cfg, userName)
|
||||
var err *IamError
|
||||
response, err = iama.DeleteUser(s3cfg, userName)
|
||||
if err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
// Targeted delete: one record removed and one removal propagated, plus the groups whose
|
||||
// membership changed, instead of rewriting every remaining identity.
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
if err := deleteIdentity(r.Context(), cm, userName, groups); err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
case "CreateAccessKey":
|
||||
iama.handleImplicitUsername(r, values)
|
||||
var err *IamError
|
||||
response, err = iama.CreateAccessKey(s3cfg, values)
|
||||
userName := values.Get("UserName")
|
||||
resp, err := iama.CreateAccessKey(s3cfg, values)
|
||||
if err != nil {
|
||||
glog.Errorf("CreateAccessKey: %+v", err.Error)
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
response = resp
|
||||
// Targeted write: credentialManager.CreateAccessKey adds the key to the one identity and
|
||||
// propagates that identity alone. The handler creates the identity for a user it does
|
||||
// not know; that case still needs the full save.
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
if _, known := cm.GetUser(r.Context(), userName); known == nil {
|
||||
cred := &iam_pb.Credential{
|
||||
AccessKey: *resp.CreateAccessKeyResult.AccessKey.AccessKeyId,
|
||||
SecretKey: *resp.CreateAccessKeyResult.AccessKey.SecretAccessKey,
|
||||
Status: accessKeyStatusActive,
|
||||
}
|
||||
if err := cm.CreateAccessKey(r.Context(), userName, cred); err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
}
|
||||
case "DeleteAccessKey":
|
||||
iama.handleImplicitUsername(r, values)
|
||||
userName, accessKeyId := values.Get("UserName"), values.Get("AccessKeyId")
|
||||
response = iama.DeleteAccessKey(s3cfg, values)
|
||||
// Targeted write: credentialManager.DeleteAccessKey removes the key from the one identity
|
||||
// and propagates that identity alone. The handler answers 200 for a key or user it does
|
||||
// not know, so a not-found from the store is the same no-op it always was.
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
err := cm.DeleteAccessKey(r.Context(), userName, accessKeyId)
|
||||
if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !errors.Is(err, credential.ErrUserNotFound) {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
case "UpdateAccessKey":
|
||||
iama.handleImplicitUsername(r, values)
|
||||
userName := values.Get("UserName")
|
||||
var err *IamError
|
||||
response, err = iama.UpdateAccessKey(s3cfg, values)
|
||||
if err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
case "CreatePolicy":
|
||||
var err *IamError
|
||||
response, err = iama.CreatePolicy(s3cfg, values)
|
||||
@@ -1410,6 +1559,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
// The `changed` flag is false because this does not modify the main s3cfg.Identities configuration.
|
||||
changed = false
|
||||
case "PutUserPolicy":
|
||||
userName := values.Get("UserName")
|
||||
var err *IamError
|
||||
response, err = iama.PutUserPolicy(s3cfg, values)
|
||||
if err != nil {
|
||||
@@ -1417,6 +1567,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
// The policy document is already stored through PutPolicies; what remains is the
|
||||
// identity's recomputed Actions, written and propagated for that identity alone.
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
case "GetUserPolicy":
|
||||
var err *IamError
|
||||
response, err = iama.GetUserPolicy(s3cfg, values)
|
||||
@@ -1426,12 +1585,20 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
changed = false
|
||||
case "DeleteUserPolicy":
|
||||
userName := values.Get("UserName")
|
||||
var err *IamError
|
||||
response, err = iama.DeleteUserPolicy(s3cfg, values)
|
||||
if err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, err)
|
||||
return
|
||||
}
|
||||
if cm := iama.targetedCredentialManager(userName); cm != nil {
|
||||
if err := persistIdentity(r.Context(), cm, s3cfg, userName); err != nil {
|
||||
writeIamErrorResponse(w, r, reqID, &IamError{Code: s3api.CredentialErrToIamErrCode(err), Error: err})
|
||||
return
|
||||
}
|
||||
changed, persisted = false, true
|
||||
}
|
||||
case "ListUserPolicies":
|
||||
iama.handleImplicitUsername(r, values)
|
||||
var err *IamError
|
||||
@@ -1642,6 +1809,8 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
||||
writeIamErrorResponse(w, r, reqID, &iamError)
|
||||
return
|
||||
}
|
||||
}
|
||||
if changed || persisted {
|
||||
// Reload in-memory identity maps so subsequent LookupByAccessKey calls
|
||||
// can see newly created or deleted keys immediately
|
||||
if iama.iam != nil {
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
package iamapi
|
||||
|
||||
// Targeted-write regression tests: single-identity IAM actions must write only
|
||||
// the identity they changed through the credential manager, not every identity
|
||||
// via PutS3ApiConfiguration/SaveConfiguration.
|
||||
// See https://github.com/seaweedfs/seaweedfs/issues/11865.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential/memory"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// targetedRecordingStore wraps a memory store and counts the write methods
|
||||
// the dispatcher used, so a test can assert the single-identity path was
|
||||
// taken. It embeds the concrete store (rather than the CredentialStore
|
||||
// interface) so optional interfaces like InlinePolicyStore stay visible
|
||||
// through the wrapper.
|
||||
type targetedRecordingStore struct {
|
||||
*memory.MemoryStore
|
||||
saves int
|
||||
createAccessKeys int
|
||||
deleteAccessKeys int
|
||||
updateUsers int
|
||||
deleteUsers int
|
||||
updateGroups int
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
|
||||
r.saves++
|
||||
return r.MemoryStore.SaveConfiguration(ctx, config)
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error {
|
||||
r.createAccessKeys++
|
||||
return r.MemoryStore.CreateAccessKey(ctx, username, cred)
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
|
||||
r.deleteAccessKeys++
|
||||
return r.MemoryStore.DeleteAccessKey(ctx, username, accessKey)
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error {
|
||||
r.updateUsers++
|
||||
return r.MemoryStore.UpdateUser(ctx, username, identity)
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) DeleteUser(ctx context.Context, username string) error {
|
||||
r.deleteUsers++
|
||||
return r.MemoryStore.DeleteUser(ctx, username)
|
||||
}
|
||||
|
||||
func (r *targetedRecordingStore) UpdateGroup(ctx context.Context, group *iam_pb.Group) error {
|
||||
r.updateGroups++
|
||||
return r.MemoryStore.UpdateGroup(ctx, group)
|
||||
}
|
||||
|
||||
func targetedIamApiServer() (*IamApiServer, *credential.CredentialManager, *countingConfigSaver, *targetedRecordingStore) {
|
||||
srv, cm, counter := iamApiServerWithCredentialManager()
|
||||
rec := &targetedRecordingStore{MemoryStore: cm.Store.(*memory.MemoryStore)}
|
||||
cm.Store = rec
|
||||
return srv, cm, counter, rec
|
||||
}
|
||||
|
||||
const targetedTestPolicy = `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::bench/*"]}]
|
||||
}`
|
||||
|
||||
func TestIamApiCreateAccessKeyTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
|
||||
|
||||
req, _ := iam.New(session.New()).CreateAccessKeyRequest(&iam.CreateAccessKeyInput{UserName: aws.String("alice")})
|
||||
_ = req.Build()
|
||||
out := CreateAccessKeyResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "CreateAccessKey must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 0, rec.saves)
|
||||
assert.Equal(t, 1, rec.createAccessKeys)
|
||||
|
||||
ident, err := cm.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ident.Credentials, 1)
|
||||
assert.Equal(t, *out.CreateAccessKeyResult.AccessKey.AccessKeyId, ident.Credentials[0].AccessKey)
|
||||
}
|
||||
|
||||
// CreateAccessKey for a user the store does not know still needs the full save:
|
||||
// the handler creates the identity in the config, so there is no existing
|
||||
// record to write a key into.
|
||||
func TestIamApiCreateAccessKeyUnknownUserFallsBack(t *testing.T) {
|
||||
srv, cm, counter, _ := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
|
||||
req, _ := iam.New(session.New()).CreateAccessKeyRequest(&iam.CreateAccessKeyInput{UserName: aws.String("ghost")})
|
||||
_ = req.Build()
|
||||
out := CreateAccessKeyResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 1, counter.putCalled)
|
||||
ident, err := cm.GetUser(ctx, "ghost")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, ident.Credentials, 1)
|
||||
}
|
||||
|
||||
func TestIamApiDeleteAccessKeyTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{
|
||||
Name: "alice",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake"}},
|
||||
}))
|
||||
|
||||
req, _ := iam.New(session.New()).DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String("alice"),
|
||||
AccessKeyId: aws.String("AKIATESTFAKEKEY00001"),
|
||||
})
|
||||
_ = req.Build()
|
||||
out := DeleteAccessKeyResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "DeleteAccessKey must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 1, rec.deleteAccessKeys)
|
||||
|
||||
ident, err := cm.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ident.Credentials)
|
||||
|
||||
// Deleting a key that does not exist still answers 200 and stays targeted.
|
||||
req, _ = iam.New(session.New()).DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String("alice"),
|
||||
AccessKeyId: aws.String("AKIATESTFAKEKEY99999"),
|
||||
})
|
||||
_ = req.Build()
|
||||
resp, err = executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
assert.Equal(t, 0, counter.putCalled)
|
||||
}
|
||||
|
||||
func TestIamApiUpdateAccessKeyTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{
|
||||
Name: "alice",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake", Status: "Active"}},
|
||||
}))
|
||||
|
||||
req, _ := iam.New(session.New()).UpdateAccessKeyRequest(&iam.UpdateAccessKeyInput{
|
||||
UserName: aws.String("alice"),
|
||||
AccessKeyId: aws.String("AKIATESTFAKEKEY00001"),
|
||||
Status: aws.String(iam.StatusTypeInactive),
|
||||
})
|
||||
_ = req.Build()
|
||||
out := UpdateAccessKeyResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "UpdateAccessKey must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
ident, err := cm.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ident.Credentials, 1)
|
||||
assert.Equal(t, iam.StatusTypeInactive, ident.Credentials[0].Status)
|
||||
}
|
||||
|
||||
func TestIamApiPutUserPolicyTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
|
||||
|
||||
req, _ := iam.New(session.New()).PutUserPolicyRequest(&iam.PutUserPolicyInput{
|
||||
UserName: aws.String("alice"),
|
||||
PolicyName: aws.String("read-bench"),
|
||||
PolicyDocument: aws.String(targetedTestPolicy),
|
||||
})
|
||||
_ = req.Build()
|
||||
out := PutUserPolicyResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "PutUserPolicy must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 0, rec.saves)
|
||||
assert.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
ident, err := cm.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, ident.Actions, "recomputed Actions must be persisted for the one identity")
|
||||
}
|
||||
|
||||
func TestIamApiDeleteUserPolicyTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
|
||||
|
||||
putReq, _ := iam.New(session.New()).PutUserPolicyRequest(&iam.PutUserPolicyInput{
|
||||
UserName: aws.String("alice"),
|
||||
PolicyName: aws.String("read-bench"),
|
||||
PolicyDocument: aws.String(targetedTestPolicy),
|
||||
})
|
||||
_ = putReq.Build()
|
||||
putOut := PutUserPolicyResponse{}
|
||||
resp, err := executeRequestWith(srv, putReq.HTTPRequest, &putOut)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
delReq, _ := iam.New(session.New()).DeleteUserPolicyRequest(&iam.DeleteUserPolicyInput{
|
||||
UserName: aws.String("alice"),
|
||||
PolicyName: aws.String("read-bench"),
|
||||
})
|
||||
_ = delReq.Build()
|
||||
delOut := DeleteUserPolicyResponse{}
|
||||
resp, err = executeRequestWith(srv, delReq.HTTPRequest, &delOut)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "DeleteUserPolicy must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 2, rec.updateUsers)
|
||||
|
||||
policies := Policies{}
|
||||
require.NoError(t, srv.s3ApiConfig.GetPolicies(&policies))
|
||||
assert.Empty(t, policies.InlinePolicies["alice"])
|
||||
}
|
||||
|
||||
func TestIamApiUpdateUserRenameTargeted(t *testing.T) {
|
||||
srv, cm, counter, _ := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
|
||||
require.NoError(t, cm.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
|
||||
|
||||
req, _ := iam.New(session.New()).UpdateUserRequest(&iam.UpdateUserInput{
|
||||
UserName: aws.String("alice"),
|
||||
NewUserName: aws.String("alice-renamed"),
|
||||
})
|
||||
_ = req.Build()
|
||||
out := UpdateUserResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "UpdateUser rename must not PutS3ApiConfiguration")
|
||||
|
||||
_, err = cm.GetUser(ctx, "alice")
|
||||
require.ErrorIs(t, err, credential.ErrUserNotFound)
|
||||
_, err = cm.GetUser(ctx, "alice-renamed")
|
||||
require.NoError(t, err)
|
||||
|
||||
g, err := cm.GetGroup(ctx, "devs")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"alice-renamed", "bob"}, g.Members)
|
||||
}
|
||||
|
||||
func TestIamApiDeleteUserTargeted(t *testing.T) {
|
||||
srv, cm, counter, rec := targetedIamApiServer()
|
||||
ctx := context.Background()
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}))
|
||||
require.NoError(t, cm.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}))
|
||||
require.NoError(t, cm.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
|
||||
|
||||
req, _ := iam.New(session.New()).DeleteUserRequest(&iam.DeleteUserInput{UserName: aws.String("alice")})
|
||||
_ = req.Build()
|
||||
out := DeleteUserResponse{}
|
||||
resp, err := executeRequestWith(srv, req.HTTPRequest, &out)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, resp.Code)
|
||||
|
||||
assert.Equal(t, 0, counter.putCalled, "DeleteUser must not PutS3ApiConfiguration")
|
||||
assert.Equal(t, 1, rec.deleteUsers)
|
||||
assert.Equal(t, 1, rec.updateGroups)
|
||||
|
||||
_, err = cm.GetUser(ctx, "alice")
|
||||
require.ErrorIs(t, err, credential.ErrUserNotFound)
|
||||
g, err := cm.GetGroup(ctx, "devs")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"bob"}, g.Members)
|
||||
_, err = cm.GetUser(ctx, "bob")
|
||||
require.NoError(t, err, "other identities must be untouched")
|
||||
}
|
||||
+23
-32
@@ -17,9 +17,11 @@ import (
|
||||
"github.com/jinzhu/copier"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential/memory"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/request_id"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -523,32 +525,13 @@ func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorde
|
||||
}
|
||||
|
||||
func TestHandleImplicitUsername(t *testing.T) {
|
||||
// Create a mock IamApiServer with credential store
|
||||
// The handleImplicitUsername function now looks up the username from the
|
||||
// credential store based on AccessKeyId, not from the region field in the auth header.
|
||||
// Note: Using obviously fake access keys to avoid CI secret scanner false positives
|
||||
// The implicit name is the authenticated identity recorded in the request
|
||||
// context by the auth middleware; request headers are never re-parsed.
|
||||
iama := &IamApiServer{}
|
||||
|
||||
// Create IAM directly as struct literal (same pattern as other tests)
|
||||
iam := &s3api.IdentityAccessManagement{}
|
||||
|
||||
// Load test credentials - map access key to identity name
|
||||
testConfig := &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{
|
||||
{
|
||||
Name: "testuser1",
|
||||
Credentials: []*iam_pb.Credential{
|
||||
{AccessKey: "AKIATESTFAKEKEY000001", SecretKey: "testsecretfake"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(t, testConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load test config: %v", err)
|
||||
}
|
||||
|
||||
iama := &IamApiServer{
|
||||
iam: iam,
|
||||
withIdentity := func(name string) *http.Request {
|
||||
r := &http.Request{Header: http.Header{}}
|
||||
return r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), name))
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
@@ -556,14 +539,14 @@ func TestHandleImplicitUsername(t *testing.T) {
|
||||
values url.Values
|
||||
userName string
|
||||
}{
|
||||
// No authorization header - should not set username
|
||||
// No authenticated identity - should not set username
|
||||
{&http.Request{}, url.Values{}, ""},
|
||||
// Valid auth header with known access key - should look up and find "testuser1"
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"},
|
||||
// Malformed auth header (no Credential=) - should not set username
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =AKIATESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// Unknown access key - should not set username
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// Authenticated identity in context - sets its name
|
||||
{withIdentity("testuser1"), url.Values{}, "testuser1"},
|
||||
// A client-controlled Authorization header naming another user is ignored
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// An explicit UserName is left alone
|
||||
{withIdentity("testuser1"), url.Values{"UserName": {"other"}}, "other"},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
@@ -607,6 +590,7 @@ func iamApiServerWithCredentialManager() (*IamApiServer, *credential.CredentialM
|
||||
type countingConfigSaver struct {
|
||||
cm *credential.CredentialManager
|
||||
putCalled int
|
||||
policies Policies
|
||||
}
|
||||
|
||||
func (c *countingConfigSaver) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error {
|
||||
@@ -616,6 +600,8 @@ func (c *countingConfigSaver) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigura
|
||||
}
|
||||
s3cfg.Identities = config.Identities
|
||||
s3cfg.Policies = config.Policies
|
||||
s3cfg.Groups = config.Groups
|
||||
s3cfg.ServiceAccounts = config.ServiceAccounts
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -625,10 +611,15 @@ func (c *countingConfigSaver) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfigura
|
||||
}
|
||||
|
||||
func (c *countingConfigSaver) GetPolicies(policies *Policies) error {
|
||||
*policies = c.policies
|
||||
if c.policies.Policies == nil && c.policies.InlinePolicies == nil {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *countingConfigSaver) PutPolicies(policies *Policies) error {
|
||||
c.policies = *policies
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
@@ -50,6 +51,10 @@ type StoragePreference struct {
|
||||
|
||||
type GetMasterFn func(ctx context.Context) pb.ServerAddress
|
||||
|
||||
// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
|
||||
// once; the batch assign above already fetched all their file ids.
|
||||
const submitUploadWorkers = 4
|
||||
|
||||
func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
|
||||
results := make([]SubmitResult, len(files))
|
||||
var totalBytes int64
|
||||
@@ -87,21 +92,47 @@ func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*
|
||||
file.Server = ret.PublicUrl
|
||||
}
|
||||
file.Pref = pref
|
||||
results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
|
||||
if err != nil {
|
||||
results[index].Error = err.Error()
|
||||
}
|
||||
results[index].Fid = file.Fid
|
||||
results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
|
||||
}
|
||||
return results, nil
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var lastErrLock sync.Mutex
|
||||
jobs := make(chan int)
|
||||
for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for index := range jobs {
|
||||
size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
|
||||
results[index].Size = size
|
||||
if uploadErr != nil {
|
||||
results[index].Error = uploadErr.Error()
|
||||
lastErrLock.Lock()
|
||||
err = uploadErr
|
||||
lastErrLock.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := range files {
|
||||
jobs <- index
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
return results, err
|
||||
}
|
||||
|
||||
func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
|
||||
ret = make([]*FilePart, len(fullPathFilenames))
|
||||
for index, file := range fullPathFilenames {
|
||||
if ret[index], err = newFilePart(file); err != nil {
|
||||
return
|
||||
for _, part := range ret[:index] {
|
||||
if closer, ok := part.Reader.(io.Closer); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -113,13 +144,14 @@ func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
|
||||
glog.V(0).Info("Failed to open file: ", fullPathFilename)
|
||||
return ret, openErr
|
||||
}
|
||||
ret.Reader = fh
|
||||
|
||||
fi, fiErr := fh.Stat()
|
||||
if fiErr != nil {
|
||||
fh.Close()
|
||||
glog.V(0).Info("Failed to stat file:", fullPathFilename)
|
||||
return ret, fiErr
|
||||
}
|
||||
ret.Reader = fh
|
||||
ret.ModTime = fi.ModTime().UTC().Unix()
|
||||
ret.FileSize = fi.Size()
|
||||
ext := strings.ToLower(path.Ext(fullPathFilename))
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func TestNewFilePartsClosesOnError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.txt")
|
||||
if _, err := NewFileParts([]string{missing}); err == nil {
|
||||
t.Skip("missing file unexpectedly opened")
|
||||
}
|
||||
glog.Flush()
|
||||
|
||||
fdsBefore, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
t.Skip("fd counting requires /proc")
|
||||
}
|
||||
good := filepath.Join(dir, "good.txt")
|
||||
if err := os.WriteFile(good, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := NewFileParts([]string{good, missing}); err == nil {
|
||||
t.Fatal("expected error for missing file")
|
||||
}
|
||||
fdsAfter, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
t.Skip("fd counting requires /proc")
|
||||
}
|
||||
if len(fdsAfter) != len(fdsBefore) {
|
||||
t.Errorf("fd count changed by %d: a file part leaked its handle", len(fdsAfter)-len(fdsBefore))
|
||||
}
|
||||
}
|
||||
@@ -244,7 +244,31 @@ func Mkdir(ctx context.Context, filerClient FilerClient, parentDirectoryPath str
|
||||
})
|
||||
}
|
||||
|
||||
// MkdirExclusive is Mkdir that never replaces an entry already at the path:
|
||||
// the create is O_EXCL and a taken name answers ErrEntryAlreadyExists. The
|
||||
// taken name leaves the callback as a result, not an error: failover counts a
|
||||
// callback error against the filer and retries on the next one.
|
||||
func MkdirExclusive(ctx context.Context, filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
|
||||
var taken error
|
||||
err := filerClient.WithFilerClient(false, func(client SeaweedFilerClient) error {
|
||||
err := doMkdir(ctx, client, parentDirectoryPath, dirName, true, fn)
|
||||
if errors.Is(err, ErrEntryAlreadyExists) {
|
||||
taken = err
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
err = taken
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func DoMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
|
||||
return doMkdir(ctx, client, parentDirectoryPath, dirName, false, fn)
|
||||
}
|
||||
|
||||
func doMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath string, dirName string, exclusive bool, fn func(entry *Entry)) error {
|
||||
entry := &Entry{
|
||||
Name: dirName,
|
||||
IsDirectory: true,
|
||||
@@ -264,6 +288,7 @@ func DoMkdir(ctx context.Context, client SeaweedFilerClient, parentDirectoryPath
|
||||
request := &CreateEntryRequest{
|
||||
Directory: parentDirectoryPath,
|
||||
Entry: entry,
|
||||
OExcl: exclusive,
|
||||
}
|
||||
|
||||
glog.V(1).InfofCtx(ctx, "mkdir: %v", request)
|
||||
|
||||
@@ -59,6 +59,8 @@ service Seaweed {
|
||||
}
|
||||
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
|
||||
}
|
||||
rpc ResumeVolumeGrowth (ResumeVolumeGrowthRequest) returns (ResumeVolumeGrowthResponse) {
|
||||
}
|
||||
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
|
||||
}
|
||||
rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
|
||||
@@ -119,6 +121,12 @@ message Heartbeat {
|
||||
// place of `volumes`. A master that does not understand this never sets
|
||||
// volume_digest_supported, so it keeps being sent the whole list.
|
||||
repeated VolumeInformationMessage changed_volumes = 28;
|
||||
|
||||
// Highest volume id this server has ever observed, persisted locally. Fed
|
||||
// by HeartbeatResponse.max_volume_id echoes and by AllocateVolume requests.
|
||||
uint32 max_volume_id_seen = 29;
|
||||
// Topology id of the master cluster this server last heartbeated to.
|
||||
string last_seen_topology_id = 30;
|
||||
}
|
||||
|
||||
message HeartbeatResponse {
|
||||
@@ -135,6 +143,12 @@ message HeartbeatResponse {
|
||||
// The master compares volume digests, so a server that reports one may send
|
||||
// changed_volumes in place of its whole list.
|
||||
bool volume_digest_supported = 9;
|
||||
// The leader's current allocation bound; servers persist it and report it
|
||||
// back as max_volume_id_seen so the bound survives master state loss.
|
||||
uint32 max_volume_id = 10;
|
||||
// The leader's cluster identity; a different value reported back later
|
||||
// means the master's state was rebuilt.
|
||||
string topology_id = 11;
|
||||
}
|
||||
|
||||
message VolumeInformationMessage {
|
||||
@@ -312,6 +326,13 @@ message VolumeGrowRequest {
|
||||
float grow_avg_fill_threshold = 10;
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthRequest {
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthResponse {
|
||||
bool was_frozen = 1;
|
||||
}
|
||||
|
||||
message AssignResponse {
|
||||
string fid = 1;
|
||||
uint64 count = 4;
|
||||
|
||||
+452
-318
File diff suppressed because it is too large.
Load diff
@@ -44,6 +44,7 @@ const (
|
||||
Seaweed_RaftAddServer_FullMethodName = "/master_pb.Seaweed/RaftAddServer"
|
||||
Seaweed_RaftRemoveServer_FullMethodName = "/master_pb.Seaweed/RaftRemoveServer"
|
||||
Seaweed_RaftLeadershipTransfer_FullMethodName = "/master_pb.Seaweed/RaftLeadershipTransfer"
|
||||
Seaweed_ResumeVolumeGrowth_FullMethodName = "/master_pb.Seaweed/ResumeVolumeGrowth"
|
||||
Seaweed_VolumeGrow_FullMethodName = "/master_pb.Seaweed/VolumeGrow"
|
||||
Seaweed_CollectionStatistics_FullMethodName = "/master_pb.Seaweed/CollectionStatistics"
|
||||
)
|
||||
@@ -77,6 +78,7 @@ type SeaweedClient interface {
|
||||
RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error)
|
||||
RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error)
|
||||
RaftLeadershipTransfer(ctx context.Context, in *RaftLeadershipTransferRequest, opts ...grpc.CallOption) (*RaftLeadershipTransferResponse, error)
|
||||
ResumeVolumeGrowth(ctx context.Context, in *ResumeVolumeGrowthRequest, opts ...grpc.CallOption) (*ResumeVolumeGrowthResponse, error)
|
||||
VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error)
|
||||
CollectionStatistics(ctx context.Context, in *CollectionStatisticsRequest, opts ...grpc.CallOption) (*CollectionStatisticsResponse, error)
|
||||
}
|
||||
@@ -357,6 +359,16 @@ func (c *seaweedClient) RaftLeadershipTransfer(ctx context.Context, in *RaftLead
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) ResumeVolumeGrowth(ctx context.Context, in *ResumeVolumeGrowthRequest, opts ...grpc.CallOption) (*ResumeVolumeGrowthResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ResumeVolumeGrowthResponse)
|
||||
err := c.cc.Invoke(ctx, Seaweed_ResumeVolumeGrowth_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(VolumeGrowResponse)
|
||||
@@ -406,6 +418,7 @@ type SeaweedServer interface {
|
||||
RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error)
|
||||
RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error)
|
||||
RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error)
|
||||
ResumeVolumeGrowth(context.Context, *ResumeVolumeGrowthRequest) (*ResumeVolumeGrowthResponse, error)
|
||||
VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error)
|
||||
CollectionStatistics(context.Context, *CollectionStatisticsRequest) (*CollectionStatisticsResponse, error)
|
||||
mustEmbedUnimplementedSeaweedServer()
|
||||
@@ -493,6 +506,9 @@ func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveS
|
||||
func (UnimplementedSeaweedServer) RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method RaftLeadershipTransfer not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) ResumeVolumeGrowth(context.Context, *ResumeVolumeGrowthRequest) (*ResumeVolumeGrowthResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ResumeVolumeGrowth not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeGrow not implemented")
|
||||
}
|
||||
@@ -930,6 +946,24 @@ func _Seaweed_RaftLeadershipTransfer_Handler(srv interface{}, ctx context.Contex
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_ResumeVolumeGrowth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ResumeVolumeGrowthRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedServer).ResumeVolumeGrowth(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: Seaweed_ResumeVolumeGrowth_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedServer).ResumeVolumeGrowth(ctx, req.(*ResumeVolumeGrowthRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_VolumeGrow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(VolumeGrowRequest)
|
||||
if err := dec(in); err != nil {
|
||||
@@ -1057,6 +1091,10 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "RaftLeadershipTransfer",
|
||||
Handler: _Seaweed_RaftLeadershipTransfer_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ResumeVolumeGrowth",
|
||||
Handler: _Seaweed_ResumeVolumeGrowth_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "VolumeGrow",
|
||||
Handler: _Seaweed_VolumeGrow_Handler,
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
@@ -348,6 +349,16 @@ func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocatio
|
||||
wc.ContentType = entry.Attributes.Mime
|
||||
}
|
||||
wc.ContentEncoding = remote_storage.EntryContentEncoding(entry)
|
||||
// A Writer reserves a buffer of ChunkSize bytes (16 MiB by default) so a
|
||||
// failed request can be retried. An object below that size takes a buffer
|
||||
// of its own size plus one 256 KiB unit, the SDK's granularity: a burst of
|
||||
// small uploads in flight does not hold 16 MiB each, every upload keeps
|
||||
// its retry buffer, and the object stays below ChunkSize so it goes in one
|
||||
// request rather than a resumable session. The size counts the chunks,
|
||||
// not only the attribute, which some write paths leave at zero.
|
||||
if size := filer.FileSize(entry); size < uint64(googleapi.DefaultUploadChunkSize) {
|
||||
wc.ChunkSize = int((size/googleapi.MinUploadChunkSize + 1) * googleapi.MinUploadChunkSize)
|
||||
}
|
||||
if _, err = io.Copy(wc, reader); err != nil {
|
||||
return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
|
||||
}
|
||||
|
||||
@@ -49,20 +49,29 @@ func isSourceChunkMissing(err error) bool {
|
||||
|
||||
// missingSourceChunkGate decides when to stop retrying a chunk the source cannot
|
||||
// produce, so a lookup race or a restarting volume server is waited out while a
|
||||
// vacuumed one is written off instead of retried forever. The wait is kept per
|
||||
// volume on the sink: a volume that is gone took every file it held with it, and
|
||||
// waiting it out once per file would stall the sync for as long as it holds files.
|
||||
// vacuumed one is written off instead of retried forever. A volume that is gone
|
||||
// took every file it held with it, so its wait is kept per volume on the sink;
|
||||
// a 404 is one needle's answer, so its wait is kept per file id instead — a
|
||||
// sibling needle succeeding must not restart it, and a different needle must get
|
||||
// its own full wait.
|
||||
type missingSourceChunkGate struct {
|
||||
sink *FilerSink
|
||||
volumeId string
|
||||
fileId string
|
||||
gaveUp bool
|
||||
}
|
||||
|
||||
func (g *missingSourceChunkGate) isPermanent(err error) bool {
|
||||
if !isSourceChunkMissing(err) {
|
||||
var since time.Time
|
||||
switch {
|
||||
case errors.Is(err, source.ErrVolumeNotFound):
|
||||
since = g.sink.sourceVolumeMissingSince(g.volumeId)
|
||||
case errors.Is(err, util_http.ErrNotFound):
|
||||
since = g.sink.sourceNeedleMissingSince(g.fileId)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if time.Since(g.sink.sourceVolumeMissingSince(g.volumeId)) < missingSourceChunkGrace {
|
||||
if time.Since(since) < missingSourceChunkGrace {
|
||||
return false
|
||||
}
|
||||
g.gaveUp = true
|
||||
@@ -78,7 +87,10 @@ func (g *missingSourceChunkGate) wrap(err error) error {
|
||||
return fmt.Errorf("%w: %w", errSourceChunkMissing, err)
|
||||
}
|
||||
|
||||
func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_pb.FileChunk, path string, sourceMtimeNs int64) (replicatedChunks []*filer_pb.FileChunk, err error) {
|
||||
// topLevelId anchors nested chunks to the entry-level chunk that contains them:
|
||||
// supersession is judged by whether the live entry still lists that chunk, since
|
||||
// a manifest's id is the identity of its contents.
|
||||
func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_pb.FileChunk, path string, sourceMtimeNs int64, topLevelId string) (replicatedChunks []*filer_pb.FileChunk, err error) {
|
||||
if len(sourceChunks) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -125,7 +137,11 @@ func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_
|
||||
}
|
||||
|
||||
if sourceChunk.IsChunkManifest {
|
||||
replicatedChunk, replicateErr := fs.replicateOneManifestChunk(ctx, sourceChunk, path, sourceMtimeNs)
|
||||
anchor := topLevelId
|
||||
if anchor == "" {
|
||||
anchor = sourceChunk.GetFileIdString()
|
||||
}
|
||||
replicatedChunk, replicateErr := fs.replicateOneManifestChunk(ctx, sourceChunk, path, sourceMtimeNs, anchor)
|
||||
if replicateErr != nil {
|
||||
setError(replicateErr)
|
||||
break
|
||||
@@ -143,7 +159,7 @@ func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_
|
||||
defer wg.Done()
|
||||
var replicatedChunk *filer_pb.FileChunk
|
||||
retryErr := util.Retry("replicate chunks", func() error {
|
||||
chunk, e := fs.replicateOneChunk(source, path, sourceMtimeNs)
|
||||
chunk, e := fs.replicateOneChunk(source, path, sourceMtimeNs, topLevelId)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
@@ -166,9 +182,9 @@ func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
|
||||
func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64, topLevelId string) (*filer_pb.FileChunk, error) {
|
||||
|
||||
fileId, err := fs.fetchAndWrite(sourceChunk, path, sourceMtimeNs)
|
||||
fileId, err := fs.fetchAndWrite(sourceChunk, path, sourceMtimeNs, topLevelId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy %s: %w", sourceChunk.GetFileIdString(), err)
|
||||
}
|
||||
@@ -187,7 +203,7 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, path str
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
|
||||
func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64, topLevelId string) (*filer_pb.FileChunk, error) {
|
||||
// LookupFileId can transiently fail during a write burst (volume not yet
|
||||
// registered). Retry like the data-chunk path does; the gate stops on
|
||||
// supersession or, when that cannot be checked, after a few attempts.
|
||||
@@ -201,12 +217,12 @@ func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk
|
||||
}
|
||||
resolvedChunks = rc
|
||||
return nil
|
||||
}, fs.manifestResolveRetryGate(path, sourceMtimeNs, sourceChunk.GetFileIdString(), missingGate))
|
||||
}, fs.manifestResolveRetryGate(path, sourceMtimeNs, topLevelId, missingGate))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve manifest %s: %w", sourceChunk.GetFileIdString(), missingGate.wrap(err))
|
||||
}
|
||||
|
||||
replicatedResolvedChunks, err := fs.replicateChunks(ctx, resolvedChunks, path, sourceMtimeNs)
|
||||
replicatedResolvedChunks, err := fs.replicateChunks(ctx, resolvedChunks, path, sourceMtimeNs, topLevelId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replicate manifest data chunks %s: %w", sourceChunk.GetFileIdString(), err)
|
||||
}
|
||||
@@ -266,6 +282,11 @@ func (fs *FilerSink) manifestResolveRetryGate(path string, sourceMtimeNs int64,
|
||||
chunkName, path, missingSourceChunkGrace, resolveErr)
|
||||
return false
|
||||
}
|
||||
if isSourceChunkMissing(resolveErr) && fs.sourceDroppedChunk(path, chunkName, sourceMtimeNs) {
|
||||
glog.V(0).Infof("source no longer lists manifest %s for %s, skipping the obsolete version: %v",
|
||||
chunkName, path, resolveErr)
|
||||
return false
|
||||
}
|
||||
if !isTransientResolveError(resolveErr) {
|
||||
glog.V(0).Infof("resolve manifest %s for %s: non-transient error, propagating: %v", chunkName, path, resolveErr)
|
||||
return false
|
||||
@@ -355,7 +376,7 @@ func (fs *FilerSink) uploadManifestChunk(path string, sourceMtimeNs int64, sourc
|
||||
return fileId, nil
|
||||
}
|
||||
|
||||
func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (fileId string, err error) {
|
||||
func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64, topLevelId string) (fileId string, err error) {
|
||||
uploader, err := fs.getUploader()
|
||||
if err != nil {
|
||||
glog.V(0).Infof("upload source data %v: %v", sourceChunk.GetFileIdString(), err)
|
||||
@@ -478,10 +499,21 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string,
|
||||
transferStatus.mu.Lock()
|
||||
transferStatus.LastErr = retryErr.Error()
|
||||
transferStatus.mu.Unlock()
|
||||
if isSourceChunkMissing(retryErr) && !canCheckSupersession {
|
||||
glog.V(0).Infof("source does not have %s for %s, supersession unverifiable, propagating: %v",
|
||||
sourceChunk.GetFileIdString(), path, retryErr)
|
||||
return false
|
||||
if isSourceChunkMissing(retryErr) {
|
||||
if !canCheckSupersession {
|
||||
glog.V(0).Infof("source does not have %s for %s, supersession unverifiable, propagating: %v",
|
||||
sourceChunk.GetFileIdString(), path, retryErr)
|
||||
return false
|
||||
}
|
||||
anchorId := topLevelId
|
||||
if anchorId == "" {
|
||||
anchorId = sourceChunk.GetFileIdString()
|
||||
}
|
||||
if fs.sourceDroppedChunk(path, anchorId, sourceMtimeNs) {
|
||||
glog.V(0).Infof("source no longer lists %s for %s, skipping the obsolete version: %v",
|
||||
anchorId, path, retryErr)
|
||||
return false
|
||||
}
|
||||
}
|
||||
if missingGate.isPermanent(retryErr) {
|
||||
glog.Errorf("source has not had %s for %s in %v, giving up on it: %v",
|
||||
@@ -558,7 +590,7 @@ func validateReplicatedReadSize(sourceChunk *filer_pb.FileChunk, readSize int) e
|
||||
}
|
||||
|
||||
func (fs *FilerSink) newMissingSourceChunkGate(fileId string) *missingSourceChunkGate {
|
||||
return &missingSourceChunkGate{sink: fs, volumeId: filer.VolumeId(fileId)}
|
||||
return &missingSourceChunkGate{sink: fs, volumeId: filer.VolumeId(fileId), fileId: fileId}
|
||||
}
|
||||
|
||||
// sourceVolumeMissingSince returns when the sink first found volumeId
|
||||
@@ -568,11 +600,20 @@ func (fs *FilerSink) sourceVolumeMissingSince(volumeId string) time.Time {
|
||||
return since.(time.Time)
|
||||
}
|
||||
|
||||
// sourceNeedleMissingSince returns when fileId first came back not-found from
|
||||
// an answering volume, recording now on the first sighting.
|
||||
func (fs *FilerSink) sourceNeedleMissingSince(fileId string) time.Time {
|
||||
since, _ := fs.missingNeedles.LoadOrStore(fileId, time.Now())
|
||||
return since.(time.Time)
|
||||
}
|
||||
|
||||
// sourceServed records a chunk the source did serve: the probe
|
||||
// sourceStillServesChunks re-checks, and proof its volume is locatable again.
|
||||
// sourceStillServesChunks re-checks, proof its volume is locatable again, and
|
||||
// proof the needle itself is back if it had been reported missing.
|
||||
func (fs *FilerSink) sourceServed(fileId string) {
|
||||
fs.lastServedFileId.Store(&fileId)
|
||||
fs.missingVolumes.Delete(filer.VolumeId(fileId))
|
||||
fs.missingNeedles.Delete(fileId)
|
||||
}
|
||||
|
||||
// sourceStillServesChunks reports whether the source cluster can still produce the
|
||||
@@ -611,6 +652,34 @@ func (fs *FilerSink) hasSourceNewerVersion(targetPath string, sourceMtimeNs int6
|
||||
return SourceSupersedes(context.Background(), fs.filerSource, sourcePath, sourceMtimeNs)
|
||||
}
|
||||
|
||||
// sourceDroppedChunk reports whether the live source entry still exists but no
|
||||
// longer references fileId — a version rewritten in place, which the mtime
|
||||
// supersession check cannot see. Only a live entry at least as new as the
|
||||
// replayed version can prove that; an older one means the source was rolled
|
||||
// back and says nothing about whether the chunk is still needed.
|
||||
func (fs *FilerSink) sourceDroppedChunk(targetPath string, fileId string, sourceMtimeNs int64) bool {
|
||||
sourcePath, ok := fs.targetPathToSourcePath(targetPath)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
live, _, _, err := filer_pb.GetEntry(context.Background(), fs.filerSource, sourcePath)
|
||||
if err != nil || live == nil || sourceMtimeNs <= 0 || getEntryMtimeNs(live) < sourceMtimeNs {
|
||||
return false
|
||||
}
|
||||
return !chunkFileIds(live.GetChunks())[fileId]
|
||||
}
|
||||
|
||||
// chunkFileIds indexes a chunk list by file id. The id is the immutable needle
|
||||
// identity, so the sets compare chunk content exactly — manifest chunks need no
|
||||
// resolution, and no volume reads are involved.
|
||||
func chunkFileIds(chunks []*filer_pb.FileChunk) map[string]bool {
|
||||
ids := make(map[string]bool, len(chunks))
|
||||
for _, c := range chunks {
|
||||
ids[c.GetFileIdString()] = true
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// SourceSupersedes reports whether the live source's entry at sourcePath has
|
||||
// moved past the replayed version (mtimeNs) — deleted or strictly newer — so
|
||||
// skipping it is lossless. Exported for filer.backup's non-filer-sink decision.
|
||||
|
||||
@@ -233,7 +233,7 @@ func TestFetchAndWriteRejectsZeroByteSource(t *testing.T) {
|
||||
err error
|
||||
}, 1)
|
||||
go func() {
|
||||
gotFileId, gotErr := fs.fetchAndWrite(sourceChunk, "/dst/index.bin", 0)
|
||||
gotFileId, gotErr := fs.fetchAndWrite(sourceChunk, "/dst/index.bin", 0, "")
|
||||
done <- struct {
|
||||
fileId string
|
||||
err error
|
||||
@@ -331,7 +331,7 @@ func TestReplicateChunksPreservesSizeMismatchSentinel(t *testing.T) {
|
||||
|
||||
sourceChunks := []*filer_pb.FileChunk{{FileId: fid, Size: expectedSize}}
|
||||
|
||||
_, err := fs.replicateChunks(nil, sourceChunks, "/dst/index.bin", 0)
|
||||
_, err := fs.replicateChunks(nil, sourceChunks, "/dst/index.bin", 0, "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error from replicateChunks, got nil")
|
||||
}
|
||||
@@ -441,6 +441,7 @@ type sourceFilerServer struct {
|
||||
mtime int64
|
||||
volumeUrl string
|
||||
resolvable []string
|
||||
entry *filer_pb.Entry
|
||||
}
|
||||
|
||||
func (s *sourceFilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
|
||||
@@ -456,6 +457,9 @@ func (s *sourceFilerServer) LookupVolume(ctx context.Context, req *filer_pb.Look
|
||||
}
|
||||
|
||||
func (s *sourceFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
if s.entry != nil {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: s.entry}, nil
|
||||
}
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: &filer_pb.Entry{
|
||||
Name: req.Name,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: s.mtime},
|
||||
@@ -493,6 +497,26 @@ func startSourceFiler(t *testing.T, mtime int64, volumeUrl string, resolvable ..
|
||||
return filerSrc
|
||||
}
|
||||
|
||||
func startSourceFilerWithEntry(t *testing.T, entry *filer_pb.Entry, volumeUrl string, resolvable ...string) *source.FilerSource {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
server := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(server, &sourceFilerServer{volumeUrl: volumeUrl, resolvable: resolvable, entry: entry})
|
||||
go server.Serve(listener)
|
||||
t.Cleanup(server.Stop)
|
||||
|
||||
address := listener.Addr().String()
|
||||
filerSrc := &source.FilerSource{}
|
||||
if err := filerSrc.DoInitialize(address, address, "/src", false); err != nil {
|
||||
t.Fatalf("filerSource.DoInitialize: %v", err)
|
||||
}
|
||||
filerSrc.SetGrpcDialOption(grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
return filerSrc
|
||||
}
|
||||
|
||||
// A chunk the source cluster cannot produce must stop being retried once the
|
||||
// grace period is up. Before, the retry loop ran forever: the sync job never
|
||||
// completed, so it held its slot and pinned the offset watermark at the event
|
||||
@@ -505,7 +529,11 @@ func TestFetchAndWriteStopsOnMissingSourceChunk(t *testing.T) {
|
||||
util.RetryWaitTime, missingSourceChunkGrace = prevRetryWaitTime, prevGrace
|
||||
})
|
||||
|
||||
filerSrc := startSourceFiler(t, 5, "")
|
||||
live := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,01abc", Size: 10}},
|
||||
}
|
||||
filerSrc := startSourceFilerWithEntry(t, live, "")
|
||||
|
||||
fs := &FilerSink{
|
||||
filerSource: filerSrc,
|
||||
@@ -516,7 +544,7 @@ func TestFetchAndWriteStopsOnMissingSourceChunk(t *testing.T) {
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10}, "/dst/x.bin", 5*int64(time.Second))
|
||||
_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10}, "/dst/x.bin", 5*int64(time.Second), "")
|
||||
done <- fetchErr
|
||||
}()
|
||||
|
||||
@@ -579,6 +607,27 @@ func TestMissingSourceChunkGate(t *testing.T) {
|
||||
if _, found := fs.missingVolumes.Load("5617"); found {
|
||||
t.Fatal("a served chunk must clear its volume's wait")
|
||||
}
|
||||
|
||||
// a needle not-found answer keeps its own clock: the volume staying alive
|
||||
// for other needles must not restart the wait for the one that is gone
|
||||
fs.missingNeedles.Store("5619,01abc", time.Now().Add(-2*missingSourceChunkGrace))
|
||||
needleGate := fs.newMissingSourceChunkGate("5619,01abc")
|
||||
fs.sourceServed("5619,02def")
|
||||
if !needleGate.isPermanent(needleGone) {
|
||||
t.Fatal("a needle 404 must keep aging while its volume serves other needles")
|
||||
}
|
||||
|
||||
// one needle's elapsed wait must not cut short a different needle's
|
||||
fresh := fs.newMissingSourceChunkGate("5619,03ghi")
|
||||
if fresh.isPermanent(needleGone) {
|
||||
t.Fatal("a different needle on the same volume must get its own wait")
|
||||
}
|
||||
|
||||
// the needle itself serving again is the only thing that restarts its wait
|
||||
fs.sourceServed("5619,01abc")
|
||||
if needleGate.isPermanent(needleGone) {
|
||||
t.Fatal("a needle that serves again must restart its wait")
|
||||
}
|
||||
}
|
||||
|
||||
// Once the source has lost a chunk for good, holding the sync offset for its
|
||||
@@ -601,6 +650,28 @@ func TestOnReplicateChunkErrorMissingSourceChunk(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("needle 404 needs no probe", func(t *testing.T) {
|
||||
fs := &FilerSink{filerSource: startSourceFiler(t, 5, ""), dir: "/dst"}
|
||||
notFound := fmt.Errorf("copy 5617,02def: %w: %w", errSourceChunkMissing, util_http.ErrNotFound)
|
||||
|
||||
if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, notFound); err != nil {
|
||||
t.Fatalf("a 404 answered by the source is itself proof of liveness, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("proxied 404 still needs the probe", func(t *testing.T) {
|
||||
proxied := &source.FilerSource{}
|
||||
if err := proxied.DoInitialize("127.0.0.1:1", "127.0.0.1:1", "/src", true); err != nil {
|
||||
t.Fatalf("DoInitialize: %v", err)
|
||||
}
|
||||
fs := &FilerSink{filerSource: proxied, dir: "/dst"}
|
||||
notFound := fmt.Errorf("copy 5617,02def: %w: %w", errSourceChunkMissing, util_http.ErrNotFound)
|
||||
|
||||
if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, notFound); !errors.Is(err, errSourceChunkMissing) {
|
||||
t.Fatalf("a filer-proxied 404 also covers missing locations, so it must not skip without the probe, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("source locating nothing", func(t *testing.T) {
|
||||
fs := &FilerSink{filerSource: startSourceFiler(t, 5, ""), dir: "/dst"}
|
||||
served := probe
|
||||
@@ -657,7 +728,7 @@ func TestFetchAndWriteMissingSourceChunkUnverifiableSupersession(t *testing.T) {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10},
|
||||
"/backup/2026-07-10/x.bin", 5*int64(time.Second))
|
||||
"/backup/2026-07-10/x.bin", 5*int64(time.Second), "")
|
||||
done <- fetchErr
|
||||
}()
|
||||
|
||||
@@ -673,3 +744,145 @@ func TestFetchAndWriteMissingSourceChunkUnverifiableSupersession(t *testing.T) {
|
||||
t.Fatal("fetchAndWrite waited out the grace period with supersession unverifiable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceDroppedChunk(t *testing.T) {
|
||||
live := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,02def", Size: 10}},
|
||||
}
|
||||
fs := &FilerSink{filerSource: startSourceFilerWithEntry(t, live, ""), dir: "/dst"}
|
||||
|
||||
if !fs.sourceDroppedChunk("/dst/x.bin", "5617,01abc", 5*int64(time.Second)) {
|
||||
t.Fatal("live entry no longer referencing the chunk must count as dropped")
|
||||
}
|
||||
if fs.sourceDroppedChunk("/dst/x.bin", "5617,02def", 5*int64(time.Second)) {
|
||||
t.Fatal("a chunk the live entry still lists must not count as dropped")
|
||||
}
|
||||
live.Attributes.Mtime = 4
|
||||
if fs.sourceDroppedChunk("/dst/x.bin", "5617,01abc", 5*int64(time.Second)) {
|
||||
t.Fatal("a live entry older than the replayed version cannot prove the chunk dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceMovedOn(t *testing.T) {
|
||||
live := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,02def", Size: 10}},
|
||||
}
|
||||
replayed := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,01abc", Size: 10}},
|
||||
}
|
||||
|
||||
fs := &FilerSink{filerSource: startSourceFilerWithEntry(t, live, ""), dir: "/dst"}
|
||||
if !fs.sourceMovedOn("/dst/x.bin", replayed) {
|
||||
t.Fatal("same-mtime entry with different chunks is moved on")
|
||||
}
|
||||
replayed.Chunks = live.Chunks
|
||||
if fs.sourceMovedOn("/dst/x.bin", replayed) {
|
||||
t.Fatal("identical chunks at equal mtime is the same version")
|
||||
}
|
||||
live.Attributes.Mtime = 6
|
||||
if !fs.sourceMovedOn("/dst/x.bin", replayed) {
|
||||
t.Fatal("newer mtime is still moved on")
|
||||
}
|
||||
live.Attributes.Mtime = 4
|
||||
replayed.Chunks = []*filer_pb.FileChunk{{FileId: "5617,01abc", Size: 10}}
|
||||
if fs.sourceMovedOn("/dst/x.bin", replayed) {
|
||||
t.Fatal("an older live entry cannot supersede the replayed version")
|
||||
}
|
||||
}
|
||||
|
||||
// A replayed version the source rewrote in place must not wait out the
|
||||
// missing-chunk grace period: the live entry no longer lists the chunk, so the
|
||||
// loop stops on the first 404.
|
||||
func TestFetchAndWriteStopsWhenSourceDroppedChunk(t *testing.T) {
|
||||
prevRetryWaitTime := util.RetryWaitTime
|
||||
util.RetryWaitTime = 100 * time.Millisecond
|
||||
t.Cleanup(func() { util.RetryWaitTime = prevRetryWaitTime })
|
||||
|
||||
gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}))
|
||||
defer gone.Close()
|
||||
|
||||
live := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,02def", Size: 10}},
|
||||
}
|
||||
filerSrc := startSourceFilerWithEntry(t, live, strings.TrimPrefix(gone.URL, "http://"), "5617")
|
||||
|
||||
fs := &FilerSink{
|
||||
filerSource: filerSrc,
|
||||
dir: "/dst",
|
||||
executor: util.NewLimitedConcurrentExecutor(1),
|
||||
}
|
||||
fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10},
|
||||
"/dst/x.bin", 5*int64(time.Second), "")
|
||||
done <- fetchErr
|
||||
}()
|
||||
|
||||
select {
|
||||
case fetchErr := <-done:
|
||||
if !errors.Is(fetchErr, util_http.ErrNotFound) {
|
||||
t.Fatalf("expected the 404 to propagate, got %v", fetchErr)
|
||||
}
|
||||
if errors.Is(fetchErr, errSourceChunkMissing) {
|
||||
t.Fatalf("an obsolete version must not wait for the give-up gate: %v", fetchErr)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("fetchAndWrite kept retrying a chunk the live entry no longer lists")
|
||||
}
|
||||
}
|
||||
|
||||
// A missing needle that lives inside a manifest is judged by whether the live
|
||||
// entry still lists the manifest, not the child: the manifest id is the
|
||||
// identity of its contents, so a live entry that kept it still needs the child.
|
||||
// The child must keep retrying under the gate — its error must come back marked
|
||||
// permanent by the clock, not returned raw by the dropped-chunk check.
|
||||
func TestFetchAndWriteManifestChildKeepsRetrying(t *testing.T) {
|
||||
prevRetryWaitTime, prevGrace := util.RetryWaitTime, missingSourceChunkGrace
|
||||
util.RetryWaitTime = 100 * time.Millisecond
|
||||
missingSourceChunkGrace = 500 * time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
util.RetryWaitTime, missingSourceChunkGrace = prevRetryWaitTime, prevGrace
|
||||
})
|
||||
|
||||
gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}))
|
||||
defer gone.Close()
|
||||
|
||||
live := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "5617,09fed", Size: 10}},
|
||||
}
|
||||
filerSrc := startSourceFilerWithEntry(t, live, strings.TrimPrefix(gone.URL, "http://"), "5617")
|
||||
|
||||
fs := &FilerSink{
|
||||
filerSource: filerSrc,
|
||||
dir: "/dst",
|
||||
executor: util.NewLimitedConcurrentExecutor(1),
|
||||
}
|
||||
fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10},
|
||||
"/dst/x.bin", 5*int64(time.Second), "5617,09fed")
|
||||
done <- fetchErr
|
||||
}()
|
||||
|
||||
select {
|
||||
case fetchErr := <-done:
|
||||
if !errors.Is(fetchErr, errSourceChunkMissing) {
|
||||
t.Fatalf("a child of a still-listed manifest must expire via the gate, got %v", fetchErr)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("a still-listed manifest's missing child must not retry past the gate")
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
// ChunkTransferSnapshot is a lock-free, copyable view of a chunk transfer's
|
||||
@@ -67,6 +68,10 @@ type FilerSink struct {
|
||||
// sourceStillServesChunks re-checks before writing an entry off.
|
||||
lastServedFileId atomic.Pointer[string]
|
||||
missingVolumes sync.Map // source volume id -> time.Time first found unlocatable
|
||||
// missingNeedles records when each file id first came back not-found from an
|
||||
// answering volume: only that needle serving again resets it, so a live
|
||||
// volume's other needles neither shorten nor restart the wait.
|
||||
missingNeedles sync.Map // source file id -> time.Time first answered not-found
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -289,7 +294,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [
|
||||
key, len(resp.Entry.GetChunks()), len(entry.GetChunks()))
|
||||
}
|
||||
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), entry.GetChunks(), key, getEntryMtimeNs(entry))
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), entry.GetChunks(), key, getEntryMtimeNs(entry), "")
|
||||
|
||||
if err != nil {
|
||||
// don't swallow: committing mismatched bytes would propagate corruption
|
||||
@@ -369,7 +374,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
|
||||
case updateRepair:
|
||||
glog.Warningf("repair %s: destination diverged from source (dst %d chunks, src %d chunks); re-replicating full source content",
|
||||
key, len(existingEntry.GetChunks()), len(newEntry.GetChunks()))
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), newEntry.GetChunks(), targetKey, getEntryMtimeNs(newEntry))
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), newEntry.GetChunks(), targetKey, getEntryMtimeNs(newEntry), "")
|
||||
if err != nil {
|
||||
if errors.Is(err, errChunkSizeMismatch) {
|
||||
return true, fs.onCorruptChunk(targetKey, newEntry, err)
|
||||
@@ -397,7 +402,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
|
||||
}
|
||||
|
||||
// replicate the chunks that are new in the source
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), newChunks, targetKey, getEntryMtimeNs(newEntry))
|
||||
replicatedChunks, err := fs.replicateChunks(context.Background(), newChunks, targetKey, getEntryMtimeNs(newEntry), "")
|
||||
if err != nil {
|
||||
if errors.Is(err, errChunkSizeMismatch) {
|
||||
return true, fs.onCorruptChunk(targetKey, newEntry, err)
|
||||
@@ -470,7 +475,7 @@ func EntryMtimeNs(entry *filer_pb.Entry) int64 {
|
||||
// meta events are full snapshots, so a later event re-carries the current chunks).
|
||||
// Otherwise the live version is corrupt, so return the error and halt loudly.
|
||||
func (fs *FilerSink) onCorruptChunk(key string, entry *filer_pb.Entry, err error) error {
|
||||
if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
|
||||
if fs.sourceMovedOn(key, entry) {
|
||||
glog.Warningf("skip stale entry %s with superseded corrupt chunk: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
@@ -486,17 +491,51 @@ func (fs *FilerSink) onCorruptChunk(key string, entry *filer_pb.Entry, err error
|
||||
// later event from ever being checkpointed. Otherwise propagate so the offset
|
||||
// stays put and the event is retried; returning nil would silently drop the file.
|
||||
func (fs *FilerSink) onReplicateChunkError(key string, entry *filer_pb.Entry, err error) error {
|
||||
if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
|
||||
if fs.sourceMovedOn(key, entry) {
|
||||
glog.Warningf("skip stale entry %s, source superseded during replicate: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, errSourceChunkMissing) && fs.sourceStillServesChunks() {
|
||||
// A volume server's own 404 names this needle, so a direct-read 404 needs no
|
||||
// liveness probe; a filer-proxied 404 also covers "no volume locations" and
|
||||
// could be an outage, so it still must pass the probe.
|
||||
direct404 := errors.Is(err, util_http.ErrNotFound) && fs.filerSource != nil && !fs.filerSource.ProxyByFiler()
|
||||
if errors.Is(err, errSourceChunkMissing) && (direct404 || fs.sourceStillServesChunks()) {
|
||||
glog.Errorf("skip %s: the source cluster cannot produce its data, so it stays unreplicated: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("replicate entry chunks %s: %w", key, err)
|
||||
}
|
||||
|
||||
// sourceMovedOn reports whether the live source entry moved past the replayed
|
||||
// version: gone, strictly newer, or rewritten in place so it no longer
|
||||
// references the same chunks — the last is invisible to the mtime check. A live
|
||||
// entry older than the replayed version proves nothing about it.
|
||||
func (fs *FilerSink) sourceMovedOn(targetPath string, entry *filer_pb.Entry) bool {
|
||||
sourcePath, ok := fs.targetPathToSourcePath(targetPath)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
replayedMtimeNs := getEntryMtimeNs(entry)
|
||||
live, _, _, lookupErr := filer_pb.GetEntry(context.Background(), fs.filerSource, sourcePath)
|
||||
if sourceSupersedes(sourcePath, live, lookupErr, replayedMtimeNs) {
|
||||
return true
|
||||
}
|
||||
if lookupErr != nil || live == nil || replayedMtimeNs <= 0 || getEntryMtimeNs(live) < replayedMtimeNs {
|
||||
return false
|
||||
}
|
||||
liveIds := chunkFileIds(live.GetChunks())
|
||||
replayedIds := chunkFileIds(entry.GetChunks())
|
||||
if len(liveIds) != len(replayedIds) {
|
||||
return true
|
||||
}
|
||||
for id := range replayedIds {
|
||||
if !liveIds[id] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// updateAction decides how UpdateEntry reconciles an incoming source version with
|
||||
// the destination's current entry.
|
||||
type updateAction int
|
||||
|
||||
@@ -74,6 +74,13 @@ func (fs *FilerSource) SetFilerJwtProvider(provider security.FilerJwtProvider) {
|
||||
|
||||
// FilerJwt returns the side-specific filer API credential, or nil when the
|
||||
// process-wide jwt.filer_signing configuration applies.
|
||||
// ProxyByFiler reports whether chunk reads are proxied through the source
|
||||
// filer. A proxied 404 can also mean the filer found no volume locations, so it
|
||||
// does not isolate a missing needle the way a direct volume read's 404 does.
|
||||
func (fs *FilerSource) ProxyByFiler() bool {
|
||||
return fs.proxyByFiler
|
||||
}
|
||||
|
||||
func (fs *FilerSource) FilerJwt() security.FilerJwtProvider {
|
||||
return fs.jwtForFiler
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
@@ -1670,6 +1671,7 @@ func (iam *IdentityAccessManagement) isConfiguredAccountID(id string) bool {
|
||||
|
||||
func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
scrubInternalHeaders(r)
|
||||
if !iam.isEnabled() {
|
||||
f(w, r)
|
||||
return
|
||||
@@ -1679,6 +1681,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
|
||||
// succeeds but authZ denies, so the denied request still audits its
|
||||
// requester; a failed authN resolves no identity at all.
|
||||
identity, errCode, _ := iam.authRequestWithAuthType(r, action)
|
||||
errCode = refineDenialForBucketName(r, identity, errCode)
|
||||
if errCode != s3err.ErrNone {
|
||||
glog.V(3).Infof("auth error: %v", errCode)
|
||||
}
|
||||
@@ -1695,6 +1698,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
|
||||
// signature verification, which is mandatory for all PostPolicy uploads.
|
||||
func (iam *IdentityAccessManagement) AuthPostPolicy(f http.HandlerFunc, action Action) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
scrubInternalHeaders(r)
|
||||
if !iam.isEnabled() {
|
||||
f(w, r)
|
||||
return
|
||||
@@ -1717,6 +1721,7 @@ func (iam *IdentityAccessManagement) AuthPostPolicy(f http.HandlerFunc, action A
|
||||
}
|
||||
}
|
||||
|
||||
errCode = refineDenialForBucketName(r, identity, errCode)
|
||||
if errCode != s3err.ErrNone {
|
||||
glog.V(3).Infof("auth error: %v", errCode)
|
||||
}
|
||||
@@ -1725,6 +1730,39 @@ func (iam *IdentityAccessManagement) AuthPostPolicy(f http.HandlerFunc, action A
|
||||
}
|
||||
}
|
||||
|
||||
// refineDenialForBucketName answers 400 InvalidBucketName, as AWS S3 does,
|
||||
// when an authenticated caller is denied on a bucket whose name breaks the S3
|
||||
// naming rules. The rules are public and the answer depends on the name alone,
|
||||
// so it tells the caller nothing about which buckets exist. Authentication
|
||||
// failures (no identity resolved) and every other answer are left unchanged.
|
||||
func refineDenialForBucketName(r *http.Request, identity *Identity, errCode s3err.ErrorCode) s3err.ErrorCode {
|
||||
if errCode != s3err.ErrAccessDenied || identity == nil {
|
||||
return errCode
|
||||
}
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
if bucket == "" {
|
||||
return errCode
|
||||
}
|
||||
if !isASCIIBucketName(bucket) || s3bucket.VerifyS3BucketName(bucket) != nil {
|
||||
return s3err.ErrInvalidBucketName
|
||||
}
|
||||
return errCode
|
||||
}
|
||||
|
||||
// isASCIIBucketName reports whether name uses only the characters S3 allows
|
||||
// in a bucket name. VerifyS3BucketName checks with unicode.IsLower and
|
||||
// unicode.IsNumber, which also accept letters such as 'ü' and digits such as
|
||||
// '²' that S3 refuses.
|
||||
func isASCIIBucketName(name string) bool {
|
||||
for i := 0; i < len(name); i++ {
|
||||
c := name[i]
|
||||
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '.' || c == '-') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// recordIdentityInContext stores the authenticated identity, its name and its
|
||||
// principal ARN in the request context. An STS session's name is only an opaque
|
||||
// session subject, so the ARN is what carries the assumed role and session name
|
||||
@@ -1780,19 +1818,33 @@ func (iam *IdentityAccessManagement) AuthenticateRequest(r *http.Request) (*Iden
|
||||
return ident, err
|
||||
}
|
||||
|
||||
// scrubInternalHeaders drops internal headers only the server may set, so
|
||||
// client-supplied values cannot spoof identity claims or reach the handlers'
|
||||
// internal consumers (putToFiler persists the object-lock and version-id ones
|
||||
// into authoritative metadata; the SSE ones carry key material between handler
|
||||
// stages of a request). The server sets them itself after this point.
|
||||
func scrubInternalHeaders(r *http.Request) {
|
||||
r.Header.Del(s3_constants.SeaweedFSPrincipalHeader)
|
||||
r.Header.Del(s3_constants.SeaweedFSSessionTokenHeader)
|
||||
r.Header.Del(s3_constants.ExtObjectLockModeKey)
|
||||
r.Header.Del(s3_constants.ExtRetentionUntilDateKey)
|
||||
r.Header.Del(s3_constants.ExtLegalHoldKey)
|
||||
r.Header.Del(s3_constants.ExtVersionIdKey)
|
||||
r.Header.Del(s3_constants.SeaweedFSSSEKMSBaseIVHeader)
|
||||
r.Header.Del(s3_constants.SeaweedFSSSES3BaseIVHeader)
|
||||
r.Header.Del(s3_constants.SeaweedFSSSES3KeyDataHeader)
|
||||
}
|
||||
|
||||
func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request) (*Identity, s3err.ErrorCode, authType) {
|
||||
var identity *Identity
|
||||
var s3Err s3err.ErrorCode
|
||||
var found bool
|
||||
var amzAuthType string
|
||||
|
||||
// SECURITY: Prevent clients from spoofing internal IAM headers.
|
||||
// These headers are only set by the server after successful JWT authentication
|
||||
// and are trusted by downstream authorization (including S3Tables IAM checks
|
||||
// reached via AuthSignatureOnly). Clearing them here — the shared entry point
|
||||
// for every auth path — prevents privilege escalation via header injection.
|
||||
r.Header.Del(s3_constants.SeaweedFSPrincipalHeader)
|
||||
r.Header.Del(s3_constants.SeaweedFSSessionTokenHeader)
|
||||
// Clearing them here — the shared entry point for every auth path —
|
||||
// prevents privilege escalation via header injection, including through
|
||||
// the S3Tables checks reached via AuthSignatureOnly.
|
||||
scrubInternalHeaders(r)
|
||||
// Deferred so signature verification can still read a client-signed value.
|
||||
defer r.Header.Del(s3_constants.AmzAccountId)
|
||||
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
invalidNameTestAccessKey = "INVALIDNAMEKEY000001"
|
||||
invalidNameTestSecretKey = "invalid-name-secret-for-tests-only"
|
||||
invalidNameTestBucket = "readable-bucket"
|
||||
)
|
||||
|
||||
// newInvalidNameIAM has one identity that may read invalidNameTestBucket only
|
||||
// and a configured anonymous identity with no actions.
|
||||
func newInvalidNameIAM(t *testing.T) *IdentityAccessManagement {
|
||||
t.Helper()
|
||||
iam := &IdentityAccessManagement{hashes: make(map[string]*sync.Pool), hashCounters: make(map[string]*int32)}
|
||||
require.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{
|
||||
{Name: "reader", Actions: []string{"Read:" + invalidNameTestBucket},
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: invalidNameTestAccessKey, SecretKey: invalidNameTestSecretKey}}},
|
||||
{Name: s3_constants.AccountAnonymousId},
|
||||
},
|
||||
}))
|
||||
require.True(t, iam.isEnabled())
|
||||
return iam
|
||||
}
|
||||
|
||||
func invalidNameRequest(t *testing.T, bucket string, sign bool) *http.Request {
|
||||
t.Helper()
|
||||
req := mustNewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:8333/%s/key", bucket), 0, nil, t)
|
||||
if sign {
|
||||
require.NoError(t, signRequestV4(req, invalidNameTestAccessKey, invalidNameTestSecretKey))
|
||||
}
|
||||
return mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": "key"})
|
||||
}
|
||||
|
||||
func serveAuth(iam *IdentityAccessManagement, r *http.Request) (int, bool) {
|
||||
ran := false
|
||||
rec := httptest.NewRecorder()
|
||||
iam.Auth(func(w http.ResponseWriter, r *http.Request) { ran = true; w.WriteHeader(http.StatusOK) }, s3_constants.ACTION_READ).ServeHTTP(rec, r)
|
||||
return rec.Code, ran
|
||||
}
|
||||
|
||||
// A denied request on a malformed bucket name answers 400 InvalidBucketName,
|
||||
// as AWS S3 does; a denied request on a well-formed name stays 403.
|
||||
func TestAuthDeniedInvalidBucketName(t *testing.T) {
|
||||
iam := newInvalidNameIAM(t)
|
||||
|
||||
code, ran := serveAuth(iam, invalidNameRequest(t, invalidNameTestBucket, true))
|
||||
assert.True(t, ran, "the identity's own bucket is allowed")
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
|
||||
code, ran = serveAuth(iam, invalidNameRequest(t, "other-bucket", true))
|
||||
assert.False(t, ran)
|
||||
assert.Equal(t, http.StatusForbidden, code, "a well-formed name the caller may not read stays AccessDenied")
|
||||
|
||||
for _, bucket := range []string{"Bad_Bucket", "ab", "bucket..name", "bücket"} {
|
||||
for _, sign := range []bool{true, false} {
|
||||
code, ran = serveAuth(iam, invalidNameRequest(t, bucket, sign))
|
||||
assert.False(t, ran, bucket)
|
||||
assert.Equal(t, http.StatusBadRequest, code, "bucket %q signed=%v", bucket, sign)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefineDenialForBucketName(t *testing.T) {
|
||||
reader := &Identity{Name: "reader"}
|
||||
req := func(bucket string) *http.Request {
|
||||
return mux.SetURLVars(httptest.NewRequest(http.MethodGet, "/", nil), map[string]string{"bucket": bucket})
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
bucket string
|
||||
identity *Identity
|
||||
in, want s3err.ErrorCode
|
||||
}{
|
||||
{"space in the name", "bucket name", reader, s3err.ErrAccessDenied, s3err.ErrInvalidBucketName},
|
||||
{"non-ASCII letter", "bücket", reader, s3err.ErrAccessDenied, s3err.ErrInvalidBucketName},
|
||||
{"superscript digit", "b²cket", reader, s3err.ErrAccessDenied, s3err.ErrInvalidBucketName},
|
||||
{"too short", "ab", reader, s3err.ErrAccessDenied, s3err.ErrInvalidBucketName},
|
||||
{"well-formed name", "other-bucket", reader, s3err.ErrAccessDenied, s3err.ErrAccessDenied},
|
||||
{"no bucket in the request", "", reader, s3err.ErrAccessDenied, s3err.ErrAccessDenied},
|
||||
{"allowed", "Bad_Bucket", reader, s3err.ErrNone, s3err.ErrNone},
|
||||
{"authentication failed", "Bad_Bucket", nil, s3err.ErrAccessDenied, s3err.ErrAccessDenied},
|
||||
{"bad signature", "Bad_Bucket", nil, s3err.ErrSignatureDoesNotMatch, s3err.ErrSignatureDoesNotMatch},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.want, refineDenialForBucketName(req(c.bucket), c.identity, c.in), c.name)
|
||||
}
|
||||
}
|
||||
|
||||
// signedPostForm builds a multipart POST Object request for bucket, its policy
|
||||
// signed with secret under postAuthzAccessKey.
|
||||
func signedPostForm(t *testing.T, bucket, key, secret string) *http.Request {
|
||||
t.Helper()
|
||||
now := time.Now().UTC()
|
||||
amzDate := now.Format(iso8601Format)
|
||||
day := now.Format(yyyymmdd)
|
||||
credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", postAuthzAccessKey, day, postAuthzRegion, postAuthzService)
|
||||
policyJSON := fmt.Sprintf(`{"expiration":"%s","conditions":[["eq","$bucket","%s"],["eq","$key","%s"],`+
|
||||
`["eq","$x-amz-credential","%s"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-date","%s"]]}`,
|
||||
now.Add(time.Hour).Format("2006-01-02T15:04:05.000Z"), bucket, key, credential, amzDate)
|
||||
encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON))
|
||||
signature := getSignature(getSigningKey(secret, day, postAuthzRegion, postAuthzService), encodedPolicy)
|
||||
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
for _, field := range [][2]string{{"bucket", bucket}, {"key", key}, {"x-amz-credential", credential},
|
||||
{"x-amz-algorithm", "AWS4-HMAC-SHA256"}, {"x-amz-date", amzDate}, {"policy", encodedPolicy}, {"x-amz-signature", signature}} {
|
||||
require.NoError(t, writer.WriteField(field[0], field[1]))
|
||||
}
|
||||
filePart, err := writer.CreateFormFile("file", "payload.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = filePart.Write([]byte("payload"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/"+bucket, &buf)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
return mux.SetURLVars(req, map[string]string{"bucket": bucket})
|
||||
}
|
||||
|
||||
// A browser POST upload authenticates in the handler, after AuthPostPolicy has
|
||||
// delegated it, so its denials are refined there: a verified form denied on a
|
||||
// malformed bucket name answers 400, a bad signature or a well-formed name 403.
|
||||
func TestPostPolicyDeniedInvalidBucketName(t *testing.T) {
|
||||
s3a := newPostAuthzServer(t)
|
||||
for _, bucket := range []string{"Bad_Bucket", "other-bucket"} {
|
||||
s3a.bucketRegistry.setMetadataCache(&BucketMetaData{Name: bucket})
|
||||
}
|
||||
// A second identity may write Bad_Bucket, except under denied/.
|
||||
require.NoError(t, s3a.iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{
|
||||
{Name: "tester", Actions: []string{"Write:" + postAuthzBucket},
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: postAuthzAccessKey, SecretKey: postAuthzSecretKey}}},
|
||||
},
|
||||
}))
|
||||
require.NoError(t, s3a.policyEngine.engine.SetBucketPolicy("Bad_Bucket", `{"Version":"2012-10-17","Statement":[`+
|
||||
`{"Effect":"Deny","Principal":"*","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::Bad_Bucket/denied/*"]}]}`))
|
||||
|
||||
serve := func(r *http.Request) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
s3a.iam.AuthPostPolicy(s3a.PostPolicyBucketHandler, s3_constants.ACTION_WRITE).ServeHTTP(rec, r)
|
||||
return rec
|
||||
}
|
||||
|
||||
rec := serve(signedPostForm(t, "Bad_Bucket", "k", postAuthzSecretKey))
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
|
||||
assert.Contains(t, rec.Body.String(), "InvalidBucketName")
|
||||
|
||||
rec = serve(signedPostForm(t, "Bad_Bucket", "k", "not-the-secret"))
|
||||
assert.Equal(t, http.StatusForbidden, rec.Code, rec.Body.String())
|
||||
assert.Contains(t, rec.Body.String(), "SignatureDoesNotMatch", "a form that fails verification is not refined")
|
||||
|
||||
rec = serve(signedPostForm(t, "other-bucket", "k", postAuthzSecretKey))
|
||||
assert.Equal(t, http.StatusForbidden, rec.Code, rec.Body.String())
|
||||
assert.Contains(t, rec.Body.String(), "AccessDenied")
|
||||
|
||||
// The per-key authorization after the coarse Write check is refined too.
|
||||
require.NoError(t, s3a.iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{
|
||||
{Name: "tester", Actions: []string{"Write:Bad_Bucket"},
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: postAuthzAccessKey, SecretKey: postAuthzSecretKey}}},
|
||||
},
|
||||
}))
|
||||
rec = serve(signedPostForm(t, "Bad_Bucket", "denied/k", postAuthzSecretKey))
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
|
||||
assert.Contains(t, rec.Body.String(), "InvalidBucketName")
|
||||
}
|
||||
|
||||
// Unsigned GET/HEAD object reads are authorized inside the object handlers, not
|
||||
// by Auth; their denial on a malformed bucket name answers 400 as well.
|
||||
func TestAnonymousReadDeniedInvalidBucketName(t *testing.T) {
|
||||
read := func(s3a *S3ApiServer, method, bucket string) int {
|
||||
r := httptest.NewRequest(method, "/"+bucket+"/key", nil)
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": bucket, "object": "key"})
|
||||
handler := s3a.GetObjectHandler
|
||||
if method == http.MethodHead {
|
||||
handler = s3a.HeadObjectHandler
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
s3a.AuthWithPublicRead(handler, s3_constants.ACTION_READ)(rr, r)
|
||||
return rr.Code
|
||||
}
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
s3a, _ := newAnonymousReadTestServer(t)
|
||||
s3a.iam.identityAnonymous = &Identity{Name: s3_constants.AccountAnonymousId}
|
||||
assert.Equal(t, http.StatusBadRequest, read(s3a, method, "Bad_Bucket"), method)
|
||||
assert.Equal(t, http.StatusForbidden, read(s3a, method, "other-bucket"), "%s: a well-formed missing bucket stays masked", method)
|
||||
|
||||
// No anonymous identity configured: no identity resolves, as in Auth.
|
||||
s3a, _ = newAnonymousReadTestServer(t)
|
||||
assert.Equal(t, http.StatusForbidden, read(s3a, method, "Bad_Bucket"), method)
|
||||
}
|
||||
}
|
||||
|
||||
// An existing private key and a missing key in the same malformed-name bucket
|
||||
// must answer alike, or the status code tells an anonymous caller which keys
|
||||
// exist.
|
||||
func TestAnonymousReadDeniedInvalidBucketNameExistingKey(t *testing.T) {
|
||||
const bucket = "bücket"
|
||||
read := func(s3a *S3ApiServer, method, object string) int {
|
||||
r := httptest.NewRequest(method, "/"+bucket+"/"+object, nil)
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": bucket, "object": object})
|
||||
handler := s3a.GetObjectHandler
|
||||
if method == http.MethodHead {
|
||||
handler = s3a.HeadObjectHandler
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
s3a.AuthWithPublicRead(handler, s3_constants.ACTION_READ)(rr, r)
|
||||
return rr.Code
|
||||
}
|
||||
setup := func(anonymous bool) *S3ApiServer {
|
||||
s3a, f := newAnonymousReadTestServer(t)
|
||||
if anonymous {
|
||||
s3a.iam.identityAnonymous = &Identity{Name: s3_constants.AccountAnonymousId}
|
||||
}
|
||||
s3a.bucketConfigCache.Set(bucket, &BucketConfig{Name: bucket, Ownership: s3_constants.OwnershipObjectWriter})
|
||||
f.entries["/buckets/"+bucket+"/private"] = anonymousReadEntry([]byte(`[]`))
|
||||
f.entries["/buckets/"+bucket+"/public"] = anonymousReadEntry(anonymousReadGrant("READ"))
|
||||
return s3a
|
||||
}
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
s3a := setup(true)
|
||||
assert.Equal(t, http.StatusOK, read(s3a, method, "public"), "%s: a public key is still served", method)
|
||||
assert.Equal(t, http.StatusBadRequest, read(s3a, method, "missing"), method)
|
||||
assert.Equal(t, http.StatusBadRequest, read(s3a, method, "private"), "%s: an existing private key answers like a missing one", method)
|
||||
|
||||
// No anonymous identity configured: both stay 403, as in Auth.
|
||||
s3a = setup(false)
|
||||
assert.Equal(t, http.StatusForbidden, read(s3a, method, "missing"), method)
|
||||
assert.Equal(t, http.StatusForbidden, read(s3a, method, "private"), method)
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
@@ -95,11 +94,6 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.
|
||||
return nil, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
bucket := formValues.Get("bucket")
|
||||
if !identity.CanDo(s3_constants.ACTION_WRITE, bucket, "") {
|
||||
return nil, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
policy := formValues.Get("Policy")
|
||||
if policy == "" {
|
||||
return nil, s3err.ErrMissingFields
|
||||
|
||||
@@ -858,11 +858,6 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
|
||||
return nil, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
bucket := formValues.Get("bucket")
|
||||
if !identity.CanDo(s3_constants.ACTION_WRITE, bucket, "") {
|
||||
return nil, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
// Get signing key.
|
||||
signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, credHeader.scope.service)
|
||||
|
||||
@@ -959,6 +954,12 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request, externalHost
|
||||
extractedSignedHeaders[header] = []string{hostHeaderValue}
|
||||
continue
|
||||
}
|
||||
// The Go server strips Content-Length into r.ContentLength; clients do
|
||||
// sign it, so rebuild the value the same way the client saw it.
|
||||
if strings.EqualFold(header, "content-length") && r.ContentLength >= 0 {
|
||||
extractedSignedHeaders[header] = []string{strconv.FormatInt(r.ContentLength, 10)}
|
||||
continue
|
||||
}
|
||||
// For all other headers we need to find them in the HTTP headers and copy them over.
|
||||
// We skip non-existent headers to be compatible with AWS signatures.
|
||||
if values, ok := reqHeaders[http.CanonicalHeaderKey(header)]; ok {
|
||||
|
||||
@@ -681,6 +681,22 @@ func TestExtractSignedHeadersCase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A client may sign content-length even when the request carries no body
|
||||
// (aws-cli's checksum HEAD does). The Go server strips the header into
|
||||
// r.ContentLength, so extraction rebuilds it rather than dropping it.
|
||||
func TestExtractSignedHeadersContentLength(t *testing.T) {
|
||||
r, _ := http.NewRequest("HEAD", "http://bucket.domain.com/bucket/key", nil)
|
||||
r.Host = "bucket.domain.com"
|
||||
r.ContentLength = 0
|
||||
extracted, errCode := extractSignedHeaders([]string{"host", "content-length"}, r, "")
|
||||
if errCode != s3err.ErrNone {
|
||||
t.Fatalf("extractSignedHeaders failed: %v", errCode)
|
||||
}
|
||||
if got := getCanonicalHeaders(extracted); got != "content-length:0\nhost:bucket.domain.com\n" {
|
||||
t.Fatalf("unexpected canonical headers %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A router that internally rewrites the request path (the Iceberg catalog
|
||||
// collapses a doubled /v1 prefix) must still verify the signature against
|
||||
// the path the client signed, which is kept in the request context.
|
||||
|
||||
@@ -2,8 +2,12 @@ package s3api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -83,6 +87,117 @@ func (s3a *S3ApiServer) validateTableBucketObjectPath(bucket, object string) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// volumeVerifyKey matches the object Snowflake writes, reads, lists and
|
||||
// deletes at the base of an S3-compatible external volume before it creates
|
||||
// or writes Iceberg tables there (SYSTEM$VERIFY_EXTERNAL_VOLUME, also run
|
||||
// implicitly): <base>/verify_<epoch milliseconds>_<number>, a 21-byte body.
|
||||
// A table bucket refuses every key outside the Iceberg layout, so without
|
||||
// this exception Snowflake cannot use a table bucket as an external volume.
|
||||
//
|
||||
// The exception is narrow so that it cannot be used to store data:
|
||||
// - only at the bucket root: nothing can be encoded in a prefix, written
|
||||
// into a hidden tree or take a namespace or table name;
|
||||
// - one shape: 13 digits of epoch milliseconds, then 1-10 digits;
|
||||
// - never a directory: a namespace may carry a name of that shape, and the
|
||||
// key then names its directory (volumeVerifyKeyIsDirectory);
|
||||
// - a PUT must be dated within volumeVerifyWindow of the server's clock,
|
||||
// declare a Content-Length of at most volumeVerifyMaxBytes (no chunked or
|
||||
// undeclared body), and carry no user metadata, tagging, stored content
|
||||
// headers or extraction request (volumeVerifyPutAllowed). Reads and
|
||||
// deletes take a key of any date, so an object a check left behind can
|
||||
// still be removed.
|
||||
//
|
||||
// Only the calls the check makes (PUT, GET, HEAD, DELETE, DeleteObjects)
|
||||
// accept the key; multipart, copy, rename, ACL, tagging and POST policy keep
|
||||
// the plain layout rule.
|
||||
var volumeVerifyKey = regexp.MustCompile(`^verify_([0-9]{13})_[0-9]{1,10}$`)
|
||||
|
||||
const (
|
||||
volumeVerifyMaxBytes = 1024
|
||||
volumeVerifyWindow = time.Hour
|
||||
)
|
||||
|
||||
// volumeVerifyNow is the clock a PUT's key is checked against.
|
||||
var volumeVerifyNow = time.Now
|
||||
|
||||
// volumeVerifyKeyTime is the time a verification key is dated, and whether
|
||||
// the object has the key's shape.
|
||||
func volumeVerifyKeyTime(object string) (time.Time, bool) {
|
||||
m := volumeVerifyKey.FindStringSubmatch(strings.TrimPrefix(object, "/"))
|
||||
if m == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(m[1], 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms), true
|
||||
}
|
||||
|
||||
func isVolumeVerifyKey(object string) bool {
|
||||
_, ok := volumeVerifyKeyTime(object)
|
||||
return ok
|
||||
}
|
||||
|
||||
// volumeVerifyLookup reads the entry a verification key names.
|
||||
var volumeVerifyLookup = func(s3a *S3ApiServer, dir, name string) (*filer_pb.Entry, error) {
|
||||
return s3a.getEntry(dir, name)
|
||||
}
|
||||
|
||||
// volumeVerifyKeyIsDirectory reports whether the key names a directory (a
|
||||
// namespace), or whether that cannot be known.
|
||||
func (s3a *S3ApiServer) volumeVerifyKeyIsDirectory(bucket, object string) bool {
|
||||
entry, err := volumeVerifyLookup(s3a, s3a.bucketDir(bucket), strings.TrimPrefix(object, "/"))
|
||||
if err != nil {
|
||||
return !errors.Is(err, filer_pb.ErrNotFound)
|
||||
}
|
||||
return entry != nil && entry.IsDirectory
|
||||
}
|
||||
|
||||
// validateTableBucketObjectPathOrVerify is validateTableBucketObjectPath that
|
||||
// also accepts the external-volume verification key, and reports it so the
|
||||
// PUT handler can apply its limits.
|
||||
func (s3a *S3ApiServer) validateTableBucketObjectPathOrVerify(bucket, object string) (isVerify bool, err error) {
|
||||
if !s3a.isTableBucket(bucket) {
|
||||
return false, nil
|
||||
}
|
||||
if isVolumeVerifyKey(object) && !s3a.volumeVerifyKeyIsDirectory(bucket, object) {
|
||||
return true, nil
|
||||
}
|
||||
return false, s3a.validateTableBucketObjectPath(bucket, object)
|
||||
}
|
||||
|
||||
// volumeVerifyPutAllowed reports whether a PUT to the verification key is
|
||||
// dated now, declares a small body and stores nothing beside it.
|
||||
func volumeVerifyPutAllowed(r *http.Request, object string) bool {
|
||||
at, ok := volumeVerifyKeyTime(object)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if d := volumeVerifyNow().Sub(at); d > volumeVerifyWindow || d < -volumeVerifyWindow {
|
||||
return false
|
||||
}
|
||||
if r.ContentLength < 0 || r.ContentLength > volumeVerifyMaxBytes || len(r.TransferEncoding) > 0 {
|
||||
return false
|
||||
}
|
||||
if isExtractZip(r) {
|
||||
return false
|
||||
}
|
||||
for name, values := range r.Header {
|
||||
n := strings.ToLower(name)
|
||||
if strings.HasPrefix(n, "x-amz-meta-") || n == "x-amz-tagging" || n == "content-disposition" ||
|
||||
n == "response-content-disposition" || n == "content-language" ||
|
||||
n == "cache-control" || n == "expires" || n == "x-amz-website-redirect-location" {
|
||||
return false
|
||||
}
|
||||
// aws-chunked only frames the body; any other coding is stored.
|
||||
if n == "content-encoding" && storedContentEncoding(values) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) bucketPrefix(bucket string) string {
|
||||
return s3a.bucketDir(bucket) + "/"
|
||||
}
|
||||
|
||||
+47
-3
@@ -235,13 +235,15 @@ func buildPreflightResponse(rule *CORSRule, corsReq *CORSRequest) *CORSResponse
|
||||
|
||||
// Set exposed headers
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
response.ExposeHeaders = strings.Join(rule.ExposeHeaders, ", ")
|
||||
response.ExposeHeaders = formatExposeHeaders(rule.ExposeHeaders)
|
||||
}
|
||||
|
||||
// Set max age
|
||||
if rule.MaxAgeSeconds != nil {
|
||||
response.MaxAge = strconv.Itoa(*rule.MaxAgeSeconds)
|
||||
}
|
||||
|
||||
response.AllowCredentials = true
|
||||
}
|
||||
|
||||
return response
|
||||
@@ -250,7 +252,8 @@ func buildPreflightResponse(rule *CORSRule, corsReq *CORSRequest) *CORSResponse
|
||||
// buildResponse builds a CORS response from a matching rule
|
||||
func buildResponse(rule *CORSRule, corsReq *CORSRequest) *CORSResponse {
|
||||
response := &CORSResponse{
|
||||
AllowOrigin: corsReq.Origin,
|
||||
AllowOrigin: corsReq.Origin,
|
||||
AllowCredentials: true,
|
||||
}
|
||||
|
||||
// Set allowed methods
|
||||
@@ -263,7 +266,7 @@ func buildResponse(rule *CORSRule, corsReq *CORSRequest) *CORSResponse {
|
||||
|
||||
// Set expose headers
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
response.ExposeHeaders = strings.Join(rule.ExposeHeaders, ", ")
|
||||
response.ExposeHeaders = formatExposeHeaders(rule.ExposeHeaders)
|
||||
}
|
||||
|
||||
// Set max age
|
||||
@@ -274,6 +277,47 @@ func buildResponse(rule *CORSRule, corsReq *CORSRequest) *CORSResponse {
|
||||
return response
|
||||
}
|
||||
|
||||
// defaultExposeHeaders are emitted alongside "*" so credentialed CORS requests
|
||||
// can read standard response headers ("*" is treated literally when credentials
|
||||
// are included, per the Fetch spec).
|
||||
var defaultExposeHeaders = []string{
|
||||
"ETag",
|
||||
"Content-Length",
|
||||
"Content-Type",
|
||||
"Last-Modified",
|
||||
"Content-Range",
|
||||
"x-amz-request-id",
|
||||
"x-amz-version-id",
|
||||
"x-amz-tagging-count",
|
||||
}
|
||||
|
||||
// formatExposeHeaders formats the exposed headers slice into a comma-separated string.
|
||||
func formatExposeHeaders(exposeHeaders []string) string {
|
||||
var headers []string
|
||||
hasWildcard := false
|
||||
seen := map[string]bool{}
|
||||
for _, h := range exposeHeaders {
|
||||
if h == "*" {
|
||||
hasWildcard = true
|
||||
continue
|
||||
}
|
||||
if lower := strings.ToLower(h); !seen[lower] {
|
||||
seen[lower] = true
|
||||
headers = append(headers, h)
|
||||
}
|
||||
}
|
||||
if hasWildcard {
|
||||
for _, h := range defaultExposeHeaders {
|
||||
if lower := strings.ToLower(h); !seen[lower] {
|
||||
seen[lower] = true
|
||||
headers = append(headers, h)
|
||||
}
|
||||
}
|
||||
headers = append(headers, "*")
|
||||
}
|
||||
return strings.Join(headers, ", ")
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// matchesOrigin checks if the request origin matches any allowed origin
|
||||
|
||||
@@ -405,11 +405,12 @@ func TestEvaluateRequest(t *testing.T) {
|
||||
Method: "GET",
|
||||
},
|
||||
want: &CORSResponse{
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowCredentials: true,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -421,8 +422,9 @@ func TestEvaluateRequest(t *testing.T) {
|
||||
Method: "PUT",
|
||||
},
|
||||
want: &CORSResponse{
|
||||
AllowOrigin: "http://other.com",
|
||||
AllowMethods: "PUT",
|
||||
AllowOrigin: "http://other.com",
|
||||
AllowMethods: "PUT",
|
||||
AllowCredentials: true,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -447,11 +449,12 @@ func TestEvaluateRequest(t *testing.T) {
|
||||
AccessControlRequestHeaders: []string{"Content-Type"},
|
||||
},
|
||||
want: &CORSResponse{
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowCredentials: true,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -467,7 +470,30 @@ func TestEvaluateRequest(t *testing.T) {
|
||||
},
|
||||
want: &CORSResponse{
|
||||
AllowOrigin: "http://example.com",
|
||||
// No AllowMethods or AllowHeaders because the requested header is forbidden
|
||||
// No AllowMethods, AllowHeaders, or AllowCredentials because the requested header is forbidden
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "wildcard expose headers expands for credentialed responses",
|
||||
config: &CORSConfiguration{
|
||||
CORSRules: []CORSRule{
|
||||
{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
ExposeHeaders: []string{"*"},
|
||||
},
|
||||
},
|
||||
},
|
||||
corsReq: &CORSRequest{
|
||||
Origin: "http://example.com",
|
||||
Method: "GET",
|
||||
},
|
||||
want: &CORSResponse{
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET",
|
||||
ExposeHeaders: "ETag, Content-Length, Content-Type, Last-Modified, Content-Range, x-amz-request-id, x-amz-version-id, x-amz-tagging-count, *",
|
||||
AllowCredentials: true,
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -511,18 +537,20 @@ func TestApplyHeaders(t *testing.T) {
|
||||
{
|
||||
name: "complete response",
|
||||
corsResp: &CORSResponse{
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowOrigin: "http://example.com",
|
||||
AllowMethods: "GET, POST",
|
||||
AllowHeaders: "Content-Type",
|
||||
ExposeHeaders: "ETag",
|
||||
MaxAge: "3600",
|
||||
AllowCredentials: true,
|
||||
},
|
||||
want: map[string]string{
|
||||
"Access-Control-Allow-Origin": "http://example.com",
|
||||
"Access-Control-Allow-Methods": "GET, POST",
|
||||
"Access-Control-Allow-Headers": "Content-Type",
|
||||
"Access-Control-Expose-Headers": "ETag",
|
||||
"Access-Control-Max-Age": "3600",
|
||||
"Access-Control-Allow-Origin": "http://example.com",
|
||||
"Access-Control-Allow-Methods": "GET, POST",
|
||||
"Access-Control-Allow-Headers": "Content-Type",
|
||||
"Access-Control-Expose-Headers": "ETag",
|
||||
"Access-Control-Max-Age": "3600",
|
||||
"Access-Control-Allow-Credentials": "true",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -69,7 +69,8 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
|
||||
uploadIdString := s3a.generateUploadID(*input.Key)
|
||||
|
||||
uploadIdString = uploadIdString + "_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
// a time-ordered uuid keeps uploads of one key in initiation order
|
||||
uploadIdString = uploadIdString + "_" + strings.ReplaceAll(uuid.Must(uuid.NewV7()).String(), "-", "")
|
||||
|
||||
// Validate checksum algorithm before creating the upload directory
|
||||
checksumAlgo, checksumHeaderName, checksumErrCode := detectRequestedChecksumAlgorithm(r)
|
||||
@@ -1291,56 +1292,77 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
|
||||
|
||||
glog.V(2).Infof("listMultipartUploads input %v", input)
|
||||
|
||||
output = &ListMultipartUploadsResult{
|
||||
Bucket: input.Bucket,
|
||||
Delimiter: input.Delimiter,
|
||||
EncodingType: input.EncodingType,
|
||||
KeyMarker: input.KeyMarker,
|
||||
MaxUploads: input.MaxUploads,
|
||||
Prefix: input.Prefix,
|
||||
IsTruncated: aws.Bool(false),
|
||||
}
|
||||
|
||||
entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", *input.UploadIdMarker, false, math.MaxInt32)
|
||||
entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", "", false, math.MaxInt32)
|
||||
if err != nil {
|
||||
// A missing .uploads folder normally lists as empty with no error; a
|
||||
// store that reports it as not-found still means an empty list.
|
||||
if isFilerNotFound(err) {
|
||||
return output, s3err.ErrNone
|
||||
return selectMultipartUploads(input, nil), s3err.ErrNone
|
||||
}
|
||||
// surface a real store error; a masked empty 200 makes a resuming client treat the upload as gone
|
||||
glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
|
||||
return output, s3err.ErrInternalError
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
uploadsCount := int64(0)
|
||||
return selectMultipartUploads(input, entries), s3err.ErrNone
|
||||
}
|
||||
|
||||
// selectMultipartUploads orders uploads by key, then by upload id, and resumes
|
||||
// after the (key-marker, upload-id-marker) position, so an aborted marker
|
||||
// upload does not hide the rest of its key.
|
||||
func selectMultipartUploads(input *s3.ListMultipartUploadsInput, entries []*filer_pb.Entry) *ListMultipartUploadsResult {
|
||||
output := &ListMultipartUploadsResult{
|
||||
Bucket: input.Bucket,
|
||||
Delimiter: input.Delimiter,
|
||||
EncodingType: input.EncodingType,
|
||||
KeyMarker: input.KeyMarker,
|
||||
MaxUploads: input.MaxUploads,
|
||||
Prefix: input.Prefix,
|
||||
UploadIdMarker: input.UploadIdMarker,
|
||||
IsTruncated: aws.Bool(false),
|
||||
}
|
||||
if *input.MaxUploads <= 0 {
|
||||
return output
|
||||
}
|
||||
type candidate struct {
|
||||
key string
|
||||
entry *filer_pb.Entry
|
||||
}
|
||||
var candidates []candidate
|
||||
for _, entry := range entries {
|
||||
if entry.Extended != nil {
|
||||
key := string(entry.Extended[s3_constants.ExtMultipartObjectKey])
|
||||
if *input.KeyMarker != "" && *input.KeyMarker != key {
|
||||
continue
|
||||
}
|
||||
if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {
|
||||
continue
|
||||
}
|
||||
upload := &s3.MultipartUpload{
|
||||
Key: objectKey(aws.String(key)),
|
||||
UploadId: aws.String(entry.Name),
|
||||
}
|
||||
if entry.Attributes != nil {
|
||||
upload.Initiated = aws.Time(time.Unix(entry.Attributes.Crtime, int64(entry.Attributes.CrtimeNs)))
|
||||
}
|
||||
output.Upload = append(output.Upload, upload)
|
||||
uploadsCount += 1
|
||||
if !isMultipartUploadEntry(entry) {
|
||||
continue
|
||||
}
|
||||
if uploadsCount >= *input.MaxUploads {
|
||||
output.IsTruncated = aws.Bool(true)
|
||||
output.NextUploadIdMarker = aws.String(entry.Name)
|
||||
break
|
||||
key := *objectKey(aws.String(string(entry.Extended[s3_constants.ExtMultipartObjectKey])))
|
||||
if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {
|
||||
continue
|
||||
}
|
||||
if *input.KeyMarker != "" && (key < *input.KeyMarker || key == *input.KeyMarker && (*input.UploadIdMarker == "" || entry.Name <= *input.UploadIdMarker)) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, candidate{key: key, entry: entry})
|
||||
}
|
||||
slices.SortFunc(candidates, func(a, b candidate) int {
|
||||
return cmp.Or(strings.Compare(a.key, b.key), strings.Compare(a.entry.Name, b.entry.Name))
|
||||
})
|
||||
|
||||
return
|
||||
for _, c := range candidates {
|
||||
if int64(len(output.Upload)) >= *input.MaxUploads {
|
||||
last := output.Upload[len(output.Upload)-1]
|
||||
output.IsTruncated = aws.Bool(true)
|
||||
output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId
|
||||
return output
|
||||
}
|
||||
upload := &s3.MultipartUpload{
|
||||
Key: aws.String(c.key),
|
||||
UploadId: aws.String(c.entry.Name),
|
||||
}
|
||||
if c.entry.Attributes != nil {
|
||||
upload.Initiated = aws.Time(time.Unix(c.entry.Attributes.Crtime, int64(c.entry.Attributes.CrtimeNs)))
|
||||
}
|
||||
output.Upload = append(output.Upload, upload)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
type ListPartsResult struct {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// uploadEntry is an upload folder as createMultipartUpload leaves it.
|
||||
func uploadEntry(id, key string, crtime int64) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: id,
|
||||
IsDirectory: true,
|
||||
Extended: map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte(key)},
|
||||
Attributes: &filer_pb.FuseAttributes{Crtime: crtime},
|
||||
}
|
||||
}
|
||||
|
||||
func listUploadsInput(prefix, keyMarker, uploadIdMarker string, maxUploads int64) *s3.ListMultipartUploadsInput {
|
||||
return &s3.ListMultipartUploadsInput{
|
||||
Prefix: aws.String(prefix),
|
||||
KeyMarker: aws.String(keyMarker),
|
||||
UploadIdMarker: aws.String(uploadIdMarker),
|
||||
MaxUploads: aws.Int64(maxUploads),
|
||||
}
|
||||
}
|
||||
|
||||
func listedUploads(entries []*filer_pb.Entry, prefix, keyMarker, uploadIdMarker string, maxUploads int64) ([]string, bool) {
|
||||
out := selectMultipartUploads(listUploadsInput(prefix, keyMarker, uploadIdMarker, maxUploads), entries)
|
||||
var got []string
|
||||
for _, u := range out.Upload {
|
||||
got = append(got, *u.Key+"/"+*u.UploadId)
|
||||
}
|
||||
return got, *out.IsTruncated
|
||||
}
|
||||
|
||||
func TestSelectMultipartUploads(t *testing.T) {
|
||||
// listed from the filer in upload id order, which is not key order
|
||||
entries := []*filer_pb.Entry{
|
||||
uploadEntry("1", "c", 10),
|
||||
uploadEntry("2", "a", 30),
|
||||
uploadEntry("3", "b", 10),
|
||||
uploadEntry("4", "a", 20),
|
||||
uploadEntry("5", "up/x", 10),
|
||||
// a folder with Extended set but no object key is not an upload
|
||||
{Name: "6", IsDirectory: true, Extended: map[string][]byte{}},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
keyMarker string
|
||||
uploadIdMarker string
|
||||
maxUploads int64
|
||||
want []string
|
||||
truncated bool
|
||||
}{
|
||||
{name: "sorted by key, then by upload id", maxUploads: 1000, want: []string{"a/2", "a/4", "b/3", "c/1", "up/x/5"}},
|
||||
{name: "key-marker starts after that key", keyMarker: "a", maxUploads: 1000, want: []string{"b/3", "c/1", "up/x/5"}},
|
||||
{name: "key-marker between keys", keyMarker: "bb", maxUploads: 1000, want: []string{"c/1", "up/x/5"}},
|
||||
{name: "upload-id-marker continues inside its key", keyMarker: "a", uploadIdMarker: "2", maxUploads: 1000, want: []string{"a/4", "b/3", "c/1", "up/x/5"}},
|
||||
{name: "upload-id-marker of an aborted upload", keyMarker: "a", uploadIdMarker: "3", maxUploads: 1000, want: []string{"a/4", "b/3", "c/1", "up/x/5"}},
|
||||
{name: "upload-id-marker without key-marker is ignored", uploadIdMarker: "4", maxUploads: 1000, want: []string{"a/2", "a/4", "b/3", "c/1", "up/x/5"}},
|
||||
{name: "prefix", prefix: "up/", maxUploads: 1000, want: []string{"up/x/5"}},
|
||||
{name: "page", maxUploads: 2, want: []string{"a/2", "a/4"}, truncated: true},
|
||||
{name: "exact last page is not truncated", keyMarker: "b", maxUploads: 2, want: []string{"c/1", "up/x/5"}},
|
||||
{name: "max-uploads 0", maxUploads: 0, want: nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, truncated := listedUploads(entries, tt.prefix, tt.keyMarker, tt.uploadIdMarker, tt.maxUploads)
|
||||
if !slices.Equal(got, tt.want) || truncated != tt.truncated {
|
||||
t.Fatalf("got %v truncated=%v, want %v truncated=%v", got, truncated, tt.want, tt.truncated)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectMultipartUploadsPaginatesEveryUploadOnce(t *testing.T) {
|
||||
var entries []*filer_pb.Entry
|
||||
for i, key := range []string{"k3", "k1", "k2", "k1", "k3", "k1", "k2"} {
|
||||
entries = append(entries, uploadEntry(string(rune('a'+i)), key, int64(i)))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
input := listUploadsInput("", "", "", 2)
|
||||
pages := 0
|
||||
for {
|
||||
out := selectMultipartUploads(input, entries)
|
||||
for _, u := range out.Upload {
|
||||
if seen[*u.UploadId] {
|
||||
t.Fatalf("upload %s listed twice", *u.UploadId)
|
||||
}
|
||||
seen[*u.UploadId] = true
|
||||
}
|
||||
pages++
|
||||
if !*out.IsTruncated {
|
||||
break
|
||||
}
|
||||
if out.NextKeyMarker == nil || out.NextUploadIdMarker == nil {
|
||||
t.Fatalf("truncated page without next markers")
|
||||
}
|
||||
input.KeyMarker, input.UploadIdMarker = out.NextKeyMarker, out.NextUploadIdMarker
|
||||
}
|
||||
if pages != 4 {
|
||||
t.Fatalf("got %d pages, want 4", pages)
|
||||
}
|
||||
if len(seen) != len(entries) {
|
||||
t.Fatalf("listed %d of %d uploads", len(seen), len(entries))
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
@@ -22,6 +23,23 @@ func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn fun
|
||||
|
||||
}
|
||||
|
||||
// mkdirExclusive never replaces an entry already at the path: PutBucket uses
|
||||
// it so a plain bucket cannot replace a table bucket created in its
|
||||
// check-to-create window.
|
||||
func (s3a *S3ApiServer) mkdirExclusive(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
|
||||
|
||||
return filer_pb.MkdirExclusive(context.Background(), s3a, parentDirectoryPath, dirName, fn)
|
||||
|
||||
}
|
||||
|
||||
// concurrentEntryIsForeign: after PutBucket's exclusive mkdir met an entry,
|
||||
// whether that entry must be answered BucketAlreadyExists without looking at
|
||||
// its owner: a table bucket (it carries no S3 owner and must never read as
|
||||
// "owned by you"), or an entry that could not be read.
|
||||
func concurrentEntryIsForeign(entry *filer_pb.Entry, err error) bool {
|
||||
return err != nil || entry == nil || s3tables.IsTableBucketEntry(entry)
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {
|
||||
|
||||
err := filer_pb.MkFile(context.Background(), s3a, parentDirectoryPath, fileName, chunks, fn)
|
||||
|
||||
@@ -397,7 +397,7 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
|
||||
if cleanupErr := s.deleteMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName); cleanupErr != nil {
|
||||
glog.V(1).Infof("Iceberg: failed to cleanup metadata file %s after update failure: %v", newMetadataLocation, cleanupErr)
|
||||
}
|
||||
glog.Errorf("Iceberg: CommitTable UpdateTable error: %v", err)
|
||||
logManagerError("CommitTable UpdateTable", err)
|
||||
writeManagerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
|
||||
return
|
||||
}
|
||||
glog.Errorf("Iceberg: CreateNamespace error: %v", err)
|
||||
logManagerError("CreateNamespace", err)
|
||||
writeManagerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -294,7 +294,10 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
|
||||
if markerErr := s.writeStageCreateMarker(r.Context(), bucketName, namespace, tableName, tableUUID, location, stagedMetadataLocation); markerErr != nil {
|
||||
glog.V(1).Infof("Iceberg: failed to persist stage-create marker for %s.%s: %v", flattenNamespacePath(namespace), tableName, markerErr)
|
||||
}
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location)
|
||||
// The staged table is not registered yet, so there is nothing to
|
||||
// authorize an update against; the create gate above is the write
|
||||
// authority for the prefix the staged table will occupy.
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location, func() bool { return true })
|
||||
result := LoadTableResult{
|
||||
MetadataLocation: metadataLocation,
|
||||
Metadata: metadata,
|
||||
@@ -395,7 +398,9 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
|
||||
glog.V(1).Infof("Iceberg: failed to cleanup stage-create markers for %s.%s after create: %v", flattenNamespacePath(namespace), tableName, markerErr)
|
||||
}
|
||||
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location)
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location, func() bool {
|
||||
return s.mayUpdateTable(r, bucketName, namespace, req.Name)
|
||||
})
|
||||
result := LoadTableResult{
|
||||
MetadataLocation: finalLocation,
|
||||
Metadata: metadata,
|
||||
@@ -590,7 +595,9 @@ func (s *Server) buildLoadTableResult(r *http.Request, getResp s3tables.GetTable
|
||||
return LoadTableResult{}, fmt.Errorf("build metadata for %s: %w", tableName, err)
|
||||
}
|
||||
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location)
|
||||
config, storageCredentials := s.buildFileIOConfig(r, location, func() bool {
|
||||
return s.mayUpdateTable(r, bucketName, namespace, tableName)
|
||||
})
|
||||
return LoadTableResult{
|
||||
MetadataLocation: getResp.MetadataLocation,
|
||||
Metadata: metadata,
|
||||
@@ -646,7 +653,7 @@ func applySnapshotsParam(r *http.Request, metadata table.Metadata) (table.Metada
|
||||
// a region on attach don't need to be told it out-of-band. See issue #9103.
|
||||
// It also returns, for a client that asked for credential vending, the
|
||||
// credentials scoped to this table.
|
||||
func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Properties, []StorageCredential) {
|
||||
func (s *Server) buildFileIOConfig(r *http.Request, location string, canWrite func() bool) (iceberg.Properties, []StorageCredential) {
|
||||
config := make(iceberg.Properties)
|
||||
if s.s3Endpoint == "" {
|
||||
return config, nil
|
||||
@@ -654,7 +661,7 @@ func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Pr
|
||||
|
||||
vended := iceberg.Properties(nil)
|
||||
if wantsVendedCredentials(r) {
|
||||
vended = s.vendCredentials(r, location)
|
||||
vended = s.vendCredentials(r, location, canWrite)
|
||||
// A client asking for vended credentials builds its storage credential
|
||||
// out of whatever comes back here and stops using the one it was
|
||||
// configured with. With nothing to vend, an endpoint on its own leaves
|
||||
@@ -687,7 +694,7 @@ func (s *Server) buildFileIOConfig(r *http.Request, location string) (iceberg.Pr
|
||||
// nil when the deployment has no vending configured or the mint failed, which
|
||||
// leaves the caller withholding the endpoint so the client keeps its own
|
||||
// credentials rather than falling back to unsigned requests.
|
||||
func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Properties {
|
||||
func (s *Server) vendCredentials(r *http.Request, location string, canWrite func() bool) iceberg.Properties {
|
||||
if s.credentialVendor == nil || location == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -699,7 +706,7 @@ func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Prope
|
||||
}
|
||||
|
||||
principal := s3_constants.GetIdentityNameFromContext(r)
|
||||
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, prefix)
|
||||
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, prefix, canWrite == nil || !canWrite())
|
||||
if err != nil {
|
||||
glog.Warningf("Iceberg: failed to vend credentials for %s: %v", location, err)
|
||||
return nil
|
||||
@@ -719,6 +726,42 @@ func (s *Server) vendCredentials(r *http.Request, location string) iceberg.Prope
|
||||
return vended
|
||||
}
|
||||
|
||||
// mayUpdateTable reports whether the caller may commit to the table the
|
||||
// credentials would cover. The session stays read-only when it may not, so
|
||||
// the vend carries no more authority than the caller holds at the catalog.
|
||||
func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace []string, tableName string) bool {
|
||||
updateReq := &s3tables.UpdateTableRequest{
|
||||
TableBucketARN: buildTableBucketARN(bucketName),
|
||||
Namespace: namespace,
|
||||
Name: tableName,
|
||||
}
|
||||
identityName := s3_constants.GetIdentityNameFromContext(r)
|
||||
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// mayUpdateView is mayUpdateTable for a view, whose commit path is UpdateView.
|
||||
func (s *Server) mayUpdateView(r *http.Request, bucketName string, namespace []string, viewName string) bool {
|
||||
updateReq := &s3tables.UpdateViewRequest{
|
||||
TableBucketARN: buildTableBucketARN(bucketName),
|
||||
Namespace: namespace,
|
||||
Name: viewName,
|
||||
}
|
||||
identityName := s3_constants.GetIdentityNameFromContext(r)
|
||||
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s.tablesManager.AuthorizeUpdateView(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Iceberg: UpdateView check for %s/%s failed, vending read-only: %v", bucketName, viewName, err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// handleTableExists checks if a table exists.
|
||||
func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
|
||||
@@ -109,7 +109,7 @@ func (s *Server) handleCommitTransaction(w http.ResponseWriter, r *http.Request)
|
||||
writeError(w, http.StatusConflict, "CommitFailedException", "Version token mismatch")
|
||||
return
|
||||
}
|
||||
glog.Errorf("Iceberg: CommitTransaction UpdateTable error: %v", err)
|
||||
logManagerError("CommitTransaction UpdateTable", err)
|
||||
writeManagerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -218,7 +218,9 @@ func (s *Server) handleCreateView(w http.ResponseWriter, r *http.Request) {
|
||||
if finalLocation == "" {
|
||||
finalLocation = metadataLocation
|
||||
}
|
||||
config, _ := s.buildFileIOConfig(r, location)
|
||||
config, _ := s.buildFileIOConfig(r, location, func() bool {
|
||||
return s.mayUpdateView(r, bucketName, namespace, req.Name)
|
||||
})
|
||||
writeLoadResult(w, http.StatusOK, ViewResponse{
|
||||
MetadataLocation: finalLocation,
|
||||
Metadata: metadata,
|
||||
@@ -349,7 +351,9 @@ func (s *Server) buildViewResponse(r *http.Request, getResp s3tables.GetViewResp
|
||||
if err != nil {
|
||||
return ViewResponse{}, fmt.Errorf("failed to parse view metadata: %w", err)
|
||||
}
|
||||
config, _ := s.buildFileIOConfig(r, tableLocationFromMetadataLocation(getResp.MetadataLocation))
|
||||
config, _ := s.buildFileIOConfig(r, tableLocationFromMetadataLocation(getResp.MetadataLocation), func() bool {
|
||||
return s.mayUpdateView(r, bucketName, namespace, viewName)
|
||||
})
|
||||
return ViewResponse{
|
||||
MetadataLocation: getResp.MetadataLocation,
|
||||
Metadata: metadata,
|
||||
|
||||
@@ -133,7 +133,7 @@ func (s *Server) handleUpdateView(w http.ResponseWriter, r *http.Request) {
|
||||
return s.tablesManager.Execute(r.Context(), mgrClient, "UpdateView", updateReq, nil, identityName)
|
||||
})
|
||||
if err == nil {
|
||||
config, _ := s.buildFileIOConfig(r, location)
|
||||
config, _ := s.buildFileIOConfig(r, location, func() bool { return true })
|
||||
writeLoadResult(w, http.StatusOK, ViewResponse{
|
||||
MetadataLocation: newMetadataLocation,
|
||||
Metadata: newMetadata,
|
||||
|
||||
@@ -43,12 +43,12 @@ func TestBuildFileIOConfigWithholdsEndpointFromCredentialVendingClients(t *testi
|
||||
|
||||
r := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
||||
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
if got, _ := s.buildFileIOConfig(r, "s3://warehouse/ns/t"); len(got) != 0 {
|
||||
if got, _ := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil); len(got) != 0 {
|
||||
t.Fatalf("buildFileIOConfig() = %v, want empty for a vended-credentials request", got)
|
||||
}
|
||||
|
||||
plain := httptest.NewRequest(http.MethodGet, "/v1/namespaces/ns/tables/t", nil)
|
||||
if got, _ := s.buildFileIOConfig(plain, "s3://warehouse/ns/t"); got["s3.endpoint"] != s.s3Endpoint {
|
||||
if got, _ := s.buildFileIOConfig(plain, "s3://warehouse/ns/t", nil); got["s3.endpoint"] != s.s3Endpoint {
|
||||
t.Fatalf("s3.endpoint = %q, want %q", got["s3.endpoint"], s.s3Endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,10 +14,11 @@ type stubVendor struct {
|
||||
gotBucket string
|
||||
gotPrefix string
|
||||
gotcaller string
|
||||
gotReadOnly bool
|
||||
}
|
||||
|
||||
func (v *stubVendor) VendTableCredentials(_ context.Context, principal, bucket, prefix string) (*VendedCredentials, error) {
|
||||
v.gotcaller, v.gotBucket, v.gotPrefix = principal, bucket, prefix
|
||||
func (v *stubVendor) VendTableCredentials(_ context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) {
|
||||
v.gotcaller, v.gotBucket, v.gotPrefix, v.gotReadOnly = principal, bucket, prefix, readOnly
|
||||
return v.credentials, v.err
|
||||
}
|
||||
|
||||
@@ -40,7 +41,7 @@ func TestBuildFileIOConfigVendingVendsScopedCredentials(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
|
||||
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
|
||||
|
||||
if vendor.gotBucket != "warehouse" || vendor.gotPrefix != "ns/t" {
|
||||
t.Errorf("vendor called with bucket=%q prefix=%q, want warehouse and ns/t", vendor.gotBucket, vendor.gotPrefix)
|
||||
@@ -70,7 +71,7 @@ func TestBuildFileIOConfigVendingWithoutVendorSaysNothing(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
|
||||
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
|
||||
if len(config) != 0 || storageCredentials != nil {
|
||||
t.Errorf("config = %v, storage-credentials = %v, want both empty", config, storageCredentials)
|
||||
}
|
||||
@@ -81,12 +82,35 @@ func TestBuildFileIOConfigVendingFallsBackWhenVendingFails(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
|
||||
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
|
||||
if len(config) != 0 || storageCredentials != nil {
|
||||
t.Errorf("config = %v, storage-credentials = %v, want both empty", config, storageCredentials)
|
||||
}
|
||||
}
|
||||
|
||||
// The vend stays read-only unless the caller proved write authority: a nil
|
||||
// or false canWrite must reach the vendor as readOnly.
|
||||
func TestBuildFileIOConfigVendingReadOnlyUnlessCanWrite(t *testing.T) {
|
||||
vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
|
||||
s := vendingServer(vendor)
|
||||
|
||||
r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
|
||||
r.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
|
||||
s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
|
||||
if !vendor.gotReadOnly {
|
||||
t.Error("vendCredentials() passed readOnly=false for a nil canWrite, want readOnly=true")
|
||||
}
|
||||
s.buildFileIOConfig(r, "s3://warehouse/ns/t", func() bool { return false })
|
||||
if !vendor.gotReadOnly {
|
||||
t.Error("vendCredentials() passed readOnly=false for canWrite=false, want readOnly=true")
|
||||
}
|
||||
s.buildFileIOConfig(r, "s3://warehouse/ns/t", func() bool { return true })
|
||||
if vendor.gotReadOnly {
|
||||
t.Error("vendCredentials() passed readOnly=true for canWrite=true, want readOnly=false")
|
||||
}
|
||||
}
|
||||
|
||||
// A client that did not ask for delegation keeps the plain endpoint config and
|
||||
// must never be handed credentials.
|
||||
func TestBuildFileIOConfigVendingWithoutDelegationHeader(t *testing.T) {
|
||||
@@ -94,7 +118,7 @@ func TestBuildFileIOConfigVendingWithoutDelegationHeader(t *testing.T) {
|
||||
s := vendingServer(vendor)
|
||||
|
||||
r := httptest.NewRequest("GET", "/v1/namespaces/ns/tables/t", nil)
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t")
|
||||
config, storageCredentials := s.buildFileIOConfig(r, "s3://warehouse/ns/t", nil)
|
||||
|
||||
if _, vended := config["s3.access-key-id"]; vended {
|
||||
t.Errorf("credentials vended without the delegation header: %v", config)
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestBuildFileIOConfig(t *testing.T) {
|
||||
|
||||
t.Run("no endpoint configured yields empty config", func(t *testing.T) {
|
||||
s := &Server{}
|
||||
got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t")
|
||||
got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t", nil)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("buildFileIOConfig() = %v, want empty", got)
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func TestBuildFileIOConfig(t *testing.T) {
|
||||
|
||||
t.Run("endpoint is advertised with path-style-access and region", func(t *testing.T) {
|
||||
s := &Server{s3Endpoint: "http://seaweed.example:8333"}
|
||||
got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t")
|
||||
got, _ := s.buildFileIOConfig(loadTable(), "s3://warehouse/ns/t", nil)
|
||||
if got["s3.endpoint"] != "http://seaweed.example:8333" {
|
||||
t.Fatalf("s3.endpoint = %q, want %q", got["s3.endpoint"], "http://seaweed.example:8333")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package iceberg
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
func TestLogManagerErrorRefusalsAreNotErrors(t *testing.T) {
|
||||
var refusals, faults []string
|
||||
oldR, oldF := logManagerRefusal, logManagerFault
|
||||
logManagerRefusal = func(f string, a ...interface{}) { refusals = append(refusals, fmt.Sprintf(f, a...)) }
|
||||
logManagerFault = func(f string, a ...interface{}) { faults = append(faults, fmt.Sprintf(f, a...)) }
|
||||
defer func() { logManagerRefusal, logManagerFault = oldR, oldF }()
|
||||
for _, c := range []struct {
|
||||
err error
|
||||
fault bool
|
||||
}{
|
||||
{&s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "not authorized to create namespace in this bucket"}, false},
|
||||
{&s3tables.S3TablesError{Type: s3tables.ErrCodeNoSuchNamespace, Message: "no such namespace"}, false},
|
||||
{&s3tables.S3TablesError{Type: s3tables.ErrCodeNamespaceAlreadyExists, Message: "exists"}, false},
|
||||
{errors.New("filer: connection refused"), true}, // positive control: a real fault stays at ERROR
|
||||
} {
|
||||
refusals, faults = nil, nil
|
||||
logManagerError("CreateNamespace", c.err)
|
||||
wantFaults, wantRefusals := 0, 1
|
||||
if c.fault {
|
||||
wantFaults, wantRefusals = 1, 0
|
||||
}
|
||||
if len(faults) != wantFaults || len(refusals) != wantRefusals {
|
||||
t.Errorf("%v: %d ERROR and %d refusal log(s), want %d and %d (refusals %v, faults %v)", c.err, len(faults), len(refusals), wantFaults, wantRefusals, refusals, faults)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,10 +45,11 @@ type VendedCredentials struct {
|
||||
}
|
||||
|
||||
// CredentialVendor mints credentials limited to a single table's prefix for a
|
||||
// caller the catalog has already authenticated and authorized. A nil result
|
||||
// with no error means the deployment has vending switched off.
|
||||
// caller the catalog has already authenticated and authorized, read-only when
|
||||
// readOnly is set. A nil result with no error means the deployment has vending
|
||||
// switched off.
|
||||
type CredentialVendor interface {
|
||||
VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
|
||||
VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error)
|
||||
}
|
||||
|
||||
// Server implements the Iceberg REST Catalog API.
|
||||
|
||||
+35
-16
@@ -158,7 +158,9 @@ func nameValidationError(err error) bool {
|
||||
// errors (400) and must carry the reason -- a variant field without
|
||||
// format-version 3 otherwise reads as a bare 500 with "variant is not supported
|
||||
// until v3" only in the log. Anything else is a server fault (500).
|
||||
func writeManagerError(w http.ResponseWriter, err error) {
|
||||
// classifyManagerError maps a storage-layer error to the HTTP status, the
|
||||
// Iceberg error type and the message the catalog answers with.
|
||||
func classifyManagerError(err error) (int, string, string) {
|
||||
switch {
|
||||
case nameValidationError(err),
|
||||
errors.Is(err, iceberg.ErrInvalidSchema),
|
||||
@@ -166,8 +168,7 @@ func writeManagerError(w http.ResponseWriter, err error) {
|
||||
errors.Is(err, iceberg.ErrInvalidTypeString),
|
||||
errors.Is(err, iceberg.ErrInvalidTransform),
|
||||
errors.Is(err, iceberg.ErrInvalidArgument):
|
||||
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
|
||||
return
|
||||
return http.StatusBadRequest, "BadRequestException", err.Error()
|
||||
}
|
||||
// Storage-layer failures are mostly the client's, not the server's. Reporting
|
||||
// a missing namespace or a name conflict as a 500 makes the catalog look
|
||||
@@ -179,31 +180,49 @@ func writeManagerError(w http.ResponseWriter, err error) {
|
||||
// The storage-layer message names the resolved bucket, which for a
|
||||
// client that sent no warehouse at all is the default one it never
|
||||
// asked for, so say how to select a real table bucket.
|
||||
writeError(w, http.StatusNotFound, "NoSuchNamespaceException",
|
||||
fmt.Sprintf("%s: each table bucket is a separate catalog, select one with warehouse=s3://<table-bucket>/ or /v1/<table-bucket>/", tableErr.Message))
|
||||
return http.StatusNotFound, "NoSuchNamespaceException",
|
||||
fmt.Sprintf("%s: each table bucket is a separate catalog, select one with warehouse=s3://<table-bucket>/ or /v1/<table-bucket>/", tableErr.Message)
|
||||
case s3tables.ErrCodeNoSuchNamespace:
|
||||
writeError(w, http.StatusNotFound, "NoSuchNamespaceException", tableErr.Message)
|
||||
return http.StatusNotFound, "NoSuchNamespaceException", tableErr.Message
|
||||
case s3tables.ErrCodeNoSuchTable:
|
||||
writeError(w, http.StatusNotFound, "NoSuchTableException", tableErr.Message)
|
||||
return http.StatusNotFound, "NoSuchTableException", tableErr.Message
|
||||
case s3tables.ErrCodeNoSuchView:
|
||||
writeError(w, http.StatusNotFound, "NoSuchViewException", tableErr.Message)
|
||||
return http.StatusNotFound, "NoSuchViewException", tableErr.Message
|
||||
case s3tables.ErrCodeNamespaceAlreadyExists, s3tables.ErrCodeBucketAlreadyExists,
|
||||
s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeViewAlreadyExists:
|
||||
writeError(w, http.StatusConflict, "AlreadyExistsException", tableErr.Message)
|
||||
return http.StatusConflict, "AlreadyExistsException", tableErr.Message
|
||||
case s3tables.ErrCodeNamespaceNotEmpty, s3tables.ErrCodeBucketNotEmpty:
|
||||
writeError(w, http.StatusConflict, "AlreadyExistsException", tableErr.Message)
|
||||
return http.StatusConflict, "AlreadyExistsException", tableErr.Message
|
||||
case s3tables.ErrCodeConflict:
|
||||
writeError(w, http.StatusConflict, "CommitFailedException", tableErr.Message)
|
||||
return http.StatusConflict, "CommitFailedException", tableErr.Message
|
||||
case s3tables.ErrCodeAccessDenied:
|
||||
writeError(w, http.StatusForbidden, "ForbiddenException", tableErr.Message)
|
||||
return http.StatusForbidden, "ForbiddenException", tableErr.Message
|
||||
case s3tables.ErrCodeInvalidRequest, s3tables.ErrCodeInvalidIcebergLayout:
|
||||
writeError(w, http.StatusBadRequest, "BadRequestException", tableErr.Message)
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
|
||||
return http.StatusBadRequest, "BadRequestException", tableErr.Message
|
||||
}
|
||||
}
|
||||
return http.StatusInternalServerError, "InternalServerError", err.Error()
|
||||
}
|
||||
|
||||
func writeManagerError(w http.ResponseWriter, err error) {
|
||||
status, typ, msg := classifyManagerError(err)
|
||||
writeError(w, status, typ, msg)
|
||||
}
|
||||
|
||||
// A storage-layer error the catalog answers with a 4xx (access denied, not
|
||||
// found, already exists, not empty, a conflict) is the API refusing a request
|
||||
// as intended, not a server fault; it is logged at V(1). A 5xx stays at ERROR.
|
||||
var (
|
||||
logManagerRefusal = func(format string, args ...interface{}) { glog.V(1).Infof(format, args...) }
|
||||
logManagerFault = glog.Errorf
|
||||
)
|
||||
|
||||
func logManagerError(op string, err error) {
|
||||
if status, _, _ := classifyManagerError(err); status >= 400 && status < 500 {
|
||||
logManagerRefusal("Iceberg: %s refused (%d): %v", op, status, err)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "InternalServerError", err.Error())
|
||||
logManagerFault("Iceberg: %s error: %v", op, err)
|
||||
}
|
||||
|
||||
// resolveWarehouseBucket maps a client-supplied warehouse value to a table
|
||||
|
||||
@@ -150,7 +150,9 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
|
||||
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
|
||||
}
|
||||
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
|
||||
// A Lance commit is a direct S3 write that never revisits the catalog, so
|
||||
// the create authorization above is the write evidence for the dataset.
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true })
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
@@ -210,7 +212,9 @@ func (s *Server) handleDescribeTable(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool {
|
||||
return s.mayUpdateTable(r, bucket, ns, name)
|
||||
})
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
|
||||
@@ -45,10 +45,11 @@ type VendedCredentials struct {
|
||||
}
|
||||
|
||||
// CredentialVendor mints credentials limited to a single table's prefix for a
|
||||
// caller the catalog has already authenticated and authorized. A nil result with
|
||||
// no error means the deployment has vending switched off.
|
||||
// caller the catalog has already authenticated and authorized, read-only when
|
||||
// readOnly is set. A nil result with no error means the deployment has vending
|
||||
// switched off.
|
||||
type CredentialVendor interface {
|
||||
VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
|
||||
VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error)
|
||||
}
|
||||
|
||||
// Server implements the Lance Namespace REST spec.
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
@@ -135,7 +136,7 @@ func confineLocation(bucket, location string) error {
|
||||
// storageOptions builds the object_store settings a Lance client needs to reach
|
||||
// the dataset. The key names are the aws_-prefixed forms Lance clients pass
|
||||
// through to object_store.
|
||||
func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool) (map[string]string, error) {
|
||||
func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool, canWrite func() bool) (map[string]string, error) {
|
||||
options := map[string]string{}
|
||||
if s.s3Endpoint != "" {
|
||||
options["aws_endpoint"] = s.s3Endpoint
|
||||
@@ -154,7 +155,7 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
|
||||
}
|
||||
|
||||
principal := s3_constants.GetIdentityNameFromContext(r)
|
||||
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location))
|
||||
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location), canWrite == nil || !canWrite())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -172,6 +173,25 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
|
||||
return options, nil
|
||||
}
|
||||
|
||||
// mayUpdateTable reports whether the caller may commit to the table the
|
||||
// credentials would cover, the same gate the Iceberg catalog applies so a
|
||||
// read-only caller never gets write-capable storage_options.
|
||||
func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool {
|
||||
req := &s3tables.UpdateTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Name: name,
|
||||
}
|
||||
identityName := s3_constants.GetIdentityNameFromContext(r)
|
||||
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), req, identityName)
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("lance: UpdateTable check for %s/%s failed, vending read-only: %v", bucket, name, err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// locationPrefix strips the s3://bucket/ part of a location, leaving the key
|
||||
// prefix a credential is scoped to.
|
||||
func locationPrefix(location string) string {
|
||||
|
||||
@@ -182,20 +182,33 @@ func resolveFromQueryParameters(query url.Values, method string, hasObject bool)
|
||||
}
|
||||
}
|
||||
|
||||
// Tagging operations
|
||||
// Tagging operations. The routes match ?tagging whenever the key is
|
||||
// present (gorilla/mux does not compare the value), so any value counts.
|
||||
// A versionId retargets the request at a specific version, so the
|
||||
// version-tagging action applies, not the current-object one.
|
||||
if query.Has("tagging") {
|
||||
versioned := hasObject && query.Get("versionId") != ""
|
||||
switch method {
|
||||
case http.MethodGet:
|
||||
if versioned {
|
||||
return s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING
|
||||
}
|
||||
if hasObject {
|
||||
return s3_constants.S3_ACTION_GET_OBJECT_TAGGING
|
||||
}
|
||||
return s3_constants.S3_ACTION_GET_BUCKET_TAGGING
|
||||
case http.MethodPut:
|
||||
if versioned {
|
||||
return s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING
|
||||
}
|
||||
if hasObject {
|
||||
return s3_constants.S3_ACTION_PUT_OBJECT_TAGGING
|
||||
}
|
||||
return s3_constants.S3_ACTION_PUT_BUCKET_TAGGING
|
||||
case http.MethodDelete:
|
||||
if versioned {
|
||||
return s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING
|
||||
}
|
||||
if hasObject {
|
||||
return s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING
|
||||
}
|
||||
|
||||
@@ -114,6 +114,38 @@ func TestResolveS3Action_AttributesBeforeVersionId(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A tagging request carrying versionId targets a specific version and must
|
||||
// resolve to the version-tagging action so a grant limited to current-object
|
||||
// tagging cannot reach historical versions.
|
||||
func TestResolveS3Action_VersionTagging(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
object string
|
||||
query string
|
||||
want string
|
||||
}{
|
||||
{"get current tags", http.MethodGet, "key", "tagging", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
|
||||
{"put current tags", http.MethodPut, "key", "tagging", s3_constants.S3_ACTION_PUT_OBJECT_TAGGING},
|
||||
{"delete current tags", http.MethodDelete, "key", "tagging", s3_constants.S3_ACTION_DELETE_OBJECT_TAGGING},
|
||||
{"get version tags", http.MethodGet, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING},
|
||||
{"put version tags", http.MethodPut, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_PUT_OBJECT_VERSION_TAGGING},
|
||||
{"delete version tags", http.MethodDelete, "key", "tagging&versionId=v1", s3_constants.S3_ACTION_DELETE_OBJECT_VERSION_TAGGING},
|
||||
{"empty versionId stays current", http.MethodGet, "key", "tagging&versionId=", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
|
||||
{"bucket tagging ignores versionId", http.MethodGet, "", "tagging&versionId=v1", s3_constants.S3_ACTION_GET_BUCKET_TAGGING},
|
||||
{"non-empty tagging is still tagging", http.MethodGet, "key", "tagging=x&versionId=v1", s3_constants.S3_ACTION_GET_OBJECT_VERSION_TAGGING},
|
||||
{"non-empty tagging plain", http.MethodGet, "key", "tagging=x", s3_constants.S3_ACTION_GET_OBJECT_TAGGING},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r, _ := http.NewRequest(tt.method, "http://localhost/bucket/"+tt.object+"?"+tt.query, nil)
|
||||
if got := ResolveS3Action(r, s3_constants.ACTION_READ, "bucket", tt.object); got != tt.want {
|
||||
t.Errorf("ResolveS3Action() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Bucket subresources registered with ACTION_ADMIN must resolve to their own
|
||||
// S3 actions so a policy granting one of them does not need s3:*, and so no
|
||||
// broader grant sweeps them in.
|
||||
|
||||
@@ -20,6 +20,11 @@ const (
|
||||
S3_ACTION_PUT_OBJECT_TAGGING = "s3:PutObjectTagging"
|
||||
S3_ACTION_DELETE_OBJECT_TAGGING = "s3:DeleteObjectTagging"
|
||||
|
||||
// Version-specific tagging operations (?tagging&versionId)
|
||||
S3_ACTION_GET_OBJECT_VERSION_TAGGING = "s3:GetObjectVersionTagging"
|
||||
S3_ACTION_PUT_OBJECT_VERSION_TAGGING = "s3:PutObjectVersionTagging"
|
||||
S3_ACTION_DELETE_OBJECT_VERSION_TAGGING = "s3:DeleteObjectVersionTagging"
|
||||
|
||||
// Object retention and legal hold
|
||||
S3_ACTION_GET_OBJECT_RETENTION = "s3:GetObjectRetention"
|
||||
S3_ACTION_PUT_OBJECT_RETENTION = "s3:PutObjectRetention"
|
||||
|
||||
@@ -330,7 +330,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
|
||||
// This ensures Object Lock configuration is set in the same CreateEntry call,
|
||||
// preventing race conditions where the bucket exists without Object Lock enabled
|
||||
var bucketCrtime int64
|
||||
if err := s3a.mkdir(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
|
||||
if err := s3a.mkdirExclusive(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
|
||||
bucketCrtime = entry.Attributes.Crtime
|
||||
|
||||
// Set bucket owner
|
||||
@@ -373,6 +373,16 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
|
||||
// return the appropriate already-exists error instead of InternalError.
|
||||
if exist, checkErr := s3a.exists(s3a.option.BucketsPath, bucket, true); checkErr == nil && exist {
|
||||
glog.V(3).Infof("PutBucketHandler: bucket %s was created concurrently", bucket)
|
||||
// From the entry itself, not the bucket registry: the pre-check
|
||||
// above has just cached this name as not found, so in the race this
|
||||
// branch exists for the registry would still say "no table bucket".
|
||||
if concurrentEntryIsForeign(s3a.getEntry(s3a.option.BucketsPath, bucket)) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrBucketAlreadyExists)
|
||||
return
|
||||
}
|
||||
if s3a.bucketConfigCache != nil {
|
||||
s3a.bucketConfigCache.RemoveNegativeCache(bucket)
|
||||
}
|
||||
errCode := s3a.existingBucketError(r, bucket, currentIdentityId, requestHasACL)
|
||||
if errCode == s3err.ErrBucketAlreadyOwnedByYou {
|
||||
s3a.healBucketOwnerIndex(bucket, currentIdentityId)
|
||||
@@ -744,7 +754,7 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
|
||||
|
||||
identityId := s3_constants.GetIdentityNameFromContext(r)
|
||||
var bucketCrtime int64
|
||||
if err := s3a.mkdir(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
|
||||
if err := s3a.mkdirExclusive(s3a.option.BucketsPath, bucket, func(entry *filer_pb.Entry) {
|
||||
bucketCrtime = entry.Attributes.Crtime
|
||||
setBucketOwner(r)(entry)
|
||||
}); err != nil {
|
||||
@@ -754,26 +764,35 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
|
||||
glog.Warningf("autoCreateBucket: failed to check existence for bucket %s: %v", bucket, err2)
|
||||
return fmt.Errorf("failed to auto-create bucket %s: %w", bucket, errors.Join(err, err2))
|
||||
} else if exist {
|
||||
entry, getErr := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if getErr != nil {
|
||||
glog.Warningf("autoCreateBucket: failed to get entry for existing bucket %s: %v", bucket, getErr)
|
||||
return fmt.Errorf("failed to auto-create bucket %s: %w", bucket, errors.Join(err, getErr))
|
||||
}
|
||||
if concurrentEntryIsForeign(entry, nil) {
|
||||
// A table bucket carries no S3 owner and must not gain one —
|
||||
// or a write intended for a missing plain bucket.
|
||||
if s3a.bucketConfigCache != nil {
|
||||
s3a.bucketConfigCache.RemoveNegativeCache(bucket)
|
||||
}
|
||||
return fmt.Errorf("auto-create bucket %s: name taken by a table bucket: %w", bucket, err)
|
||||
}
|
||||
// The bucket exists, which is fine. However, we should ensure it has an owner.
|
||||
// If it was created by a concurrent request that didn't set an owner,
|
||||
// we'll set it here to ensure consistency.
|
||||
if entry, getErr := s3a.getEntry(s3a.option.BucketsPath, bucket); getErr == nil {
|
||||
if entry.Extended == nil || len(entry.Extended[s3_constants.AmzIdentityId]) == 0 {
|
||||
// No owner set, assign current admin as owner
|
||||
setBucketOwner(r)(entry)
|
||||
if updateErr := s3a.updateEntry(s3a.option.BucketsPath, entry); updateErr != nil {
|
||||
glog.Warningf("autoCreateBucket: failed to set owner for existing bucket %s: %v", bucket, updateErr)
|
||||
} else {
|
||||
glog.V(1).Infof("Set owner for existing bucket %s (created by concurrent request)", bucket)
|
||||
if owner := bucketEntryOwner(entry); owner != "" {
|
||||
if indexErr := s3a.addBucketToOwnerIndex(owner, bucket, entry.Attributes.Crtime); indexErr != nil {
|
||||
glog.Warningf("autoCreateBucket: owner index add %s/%s: %v", owner, bucket, indexErr)
|
||||
}
|
||||
if entry.Extended == nil || len(entry.Extended[s3_constants.AmzIdentityId]) == 0 {
|
||||
// No owner set, assign current admin as owner
|
||||
setBucketOwner(r)(entry)
|
||||
if updateErr := s3a.updateEntry(s3a.option.BucketsPath, entry); updateErr != nil {
|
||||
glog.Warningf("autoCreateBucket: failed to set owner for existing bucket %s: %v", bucket, updateErr)
|
||||
} else {
|
||||
glog.V(1).Infof("Set owner for existing bucket %s (created by concurrent request)", bucket)
|
||||
if owner := bucketEntryOwner(entry); owner != "" {
|
||||
if indexErr := s3a.addBucketToOwnerIndex(owner, bucket, entry.Attributes.Crtime); indexErr != nil {
|
||||
glog.Warningf("autoCreateBucket: owner index add %s/%s: %v", owner, bucket, indexErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("autoCreateBucket: failed to get entry for existing bucket %s: %v", bucket, getErr)
|
||||
}
|
||||
// Remove bucket from negative cache — it exists now
|
||||
if s3a.bucketConfigCache != nil {
|
||||
|
||||
@@ -67,6 +67,136 @@ func (e *EmbeddedIamApi) refreshIAMConfiguration() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// canPersistIdentity reports whether a change to the named identity can be written through the
|
||||
// credential manager's single-identity methods instead of SaveConfiguration. It is false when
|
||||
// there is no credential manager, when the caller asked for no persistent write (skipPersist),
|
||||
// and for static identities, which live outside the store: those keep the full-save path.
|
||||
func (e *EmbeddedIamApi) canPersistIdentity(skipPersist bool, userName string) bool {
|
||||
return e.credentialManager != nil && !skipPersist && !e.credentialManager.IsStaticIdentity(userName)
|
||||
}
|
||||
|
||||
// groupsWithMember returns the groups in s3cfg that list userName as a member. Taken before a
|
||||
// handler renames or removes the user, it names the group records that handler will change.
|
||||
func groupsWithMember(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.Group {
|
||||
var groups []*iam_pb.Group
|
||||
for _, g := range s3cfg.Groups {
|
||||
for _, m := range g.Members {
|
||||
if m == userName {
|
||||
groups = append(groups, g)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
// serviceAccountsOf returns the service accounts in s3cfg whose parent is userName.
|
||||
func serviceAccountsOf(s3cfg *iam_pb.S3ApiConfiguration, userName string) []*iam_pb.ServiceAccount {
|
||||
var accounts []*iam_pb.ServiceAccount
|
||||
for _, sa := range s3cfg.ServiceAccounts {
|
||||
if sa.ParentUser == userName {
|
||||
accounts = append(accounts, sa)
|
||||
}
|
||||
}
|
||||
return accounts
|
||||
}
|
||||
|
||||
// persistIdentity writes the one identity a handler changed in s3cfg through the credential
|
||||
// manager. The store writes that identity's record and propagates it alone; the other
|
||||
// identities are not rewritten.
|
||||
func (e *EmbeddedIamApi) persistIdentity(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration, userName string) error {
|
||||
ident := findIdentityByName(s3cfg, userName)
|
||||
if ident == nil {
|
||||
return credential.ErrUserNotFound
|
||||
}
|
||||
return e.credentialManager.UpdateUser(ctx, userName, ident)
|
||||
}
|
||||
|
||||
// persistRenamedIdentity writes a user renamed from oldName to newName by UpdateUser through the
|
||||
// credential manager: the identity, the groups that listed the old name and the service accounts
|
||||
// it parented. Where the store has RenameUser the handler already moved the record and its inline
|
||||
// policies, and the identity is written under the new name. Otherwise UpdateUser under the old
|
||||
// name moves the record in stores that treat a changed Name as a rename (filer_etc, memory), the
|
||||
// propagating store propagates the new identity and the old name's removal, and the inline policy
|
||||
// copies the handler left under the old name are removed. A store that did neither still holds
|
||||
// the user under the old name; the caller is told so (false) and keeps the full save, which is
|
||||
// what renames such a store today.
|
||||
func (e *EmbeddedIamApi) persistRenamedIdentity(ctx context.Context, s3cfg *iam_pb.S3ApiConfiguration, oldName, newName string, groups []*iam_pb.Group, accounts []*iam_pb.ServiceAccount) (bool, error) {
|
||||
ident := findIdentityByName(s3cfg, newName)
|
||||
if ident == nil {
|
||||
return false, credential.ErrUserNotFound
|
||||
}
|
||||
storedAs := oldName
|
||||
if _, err := e.credentialManager.GetUser(ctx, newName); err == nil {
|
||||
storedAs = newName
|
||||
}
|
||||
if err := e.credentialManager.UpdateUser(ctx, storedAs, ident); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if storedAs == oldName {
|
||||
if _, err := e.credentialManager.GetUser(ctx, newName); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, oldName)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list inline policies for %s: %w", oldName, err)
|
||||
}
|
||||
for _, policyName := range policyNames {
|
||||
if err := e.credentialManager.DeleteUserInlinePolicy(ctx, oldName, policyName); err != nil {
|
||||
return false, fmt.Errorf("remove inline policy %s from %s: %w", policyName, oldName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, g := range groups {
|
||||
if err := e.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return false, fmt.Errorf("update group %s: %w", g.Name, err)
|
||||
}
|
||||
}
|
||||
for _, sa := range accounts {
|
||||
if err := e.credentialManager.UpdateServiceAccount(ctx, sa.Id, sa); err != nil {
|
||||
return false, fmt.Errorf("update service account %s: %w", sa.Id, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// deleteIdentity removes a user deleted by DeleteUser through the credential manager: its inline
|
||||
// policies, its record (the store propagates the one removal) and the groups that listed it,
|
||||
// whose membership DeleteUser already changed in s3cfg.
|
||||
func (e *EmbeddedIamApi) deleteIdentity(ctx context.Context, userName string, groups []*iam_pb.Group) error {
|
||||
policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, userName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list inline policies for %s: %w", userName, err)
|
||||
}
|
||||
for _, policyName := range policyNames {
|
||||
if err := e.credentialManager.DeleteUserInlinePolicy(ctx, userName, policyName); err != nil {
|
||||
return fmt.Errorf("remove inline policy %s from %s: %w", policyName, userName, err)
|
||||
}
|
||||
}
|
||||
if err := e.credentialManager.DeleteUser(ctx, userName); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if err := e.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("update group %s: %w", g.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reloadsAfterTargetedWrite names the actions that persist through the credential manager
|
||||
// rather than SaveConfiguration; the in-memory identity maps are refreshed after them the way
|
||||
// they are after a full save.
|
||||
func reloadsAfterTargetedWrite(action string) bool {
|
||||
switch action {
|
||||
case "AttachUserPolicy", "DetachUserPolicy", "CreatePolicy", "CreatePolicyVersion", "DeletePolicy",
|
||||
"CreateUser", "UpdateUser", "DeleteUser", "CreateAccessKey", "DeleteAccessKey", "UpdateAccessKey",
|
||||
"SetUserStatus", "PutUserPolicy", "DeleteUserPolicy", "PutGroupPolicy", "DeleteGroupPolicy":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Constants for service account identifiers
|
||||
const (
|
||||
ServiceAccountIDLength = 12 // Length of the service account ID
|
||||
@@ -2464,53 +2594,19 @@ func (e *EmbeddedIamApi) ListGroupPolicies(s3cfg *iam_pb.S3ApiConfiguration, val
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// handleImplicitUsername adds username who signs the request to values if 'username' is not specified.
|
||||
// According to AWS documentation: "If you do not specify a user name, IAM determines the user name
|
||||
// implicitly based on the Amazon Web Services access key ID signing the request."
|
||||
// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding
|
||||
// identity name in the credential store.
|
||||
// handleImplicitUsername adds the name of the authenticated caller to values
|
||||
// when 'UserName' is not specified, per AWS: "If you do not specify a user name,
|
||||
// IAM determines the user name implicitly based on the Amazon Web Services
|
||||
// access key ID signing the request." The name comes from the identity the
|
||||
// authentication middleware verified, which the request context carries; the
|
||||
// Authorization header itself is attacker-controlled and is never re-parsed.
|
||||
func (e *EmbeddedIamApi) handleImplicitUsername(r *http.Request, values url.Values) {
|
||||
if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" {
|
||||
if values.Get("UserName") != "" {
|
||||
return
|
||||
}
|
||||
// Log presence of auth header without exposing sensitive signature material
|
||||
glog.V(4).Infof("Authorization header present, extracting access key")
|
||||
// Parse AWS SigV4 Authorization header format:
|
||||
// "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..."
|
||||
s := strings.Split(r.Header["Authorization"][0], "Credential=")
|
||||
if len(s) < 2 {
|
||||
return
|
||||
if name := s3_constants.GetIdentityNameFromContext(r); name != "" {
|
||||
values.Set("UserName", name)
|
||||
}
|
||||
s = strings.Split(s[1], ",")
|
||||
if len(s) < 1 {
|
||||
return
|
||||
}
|
||||
s = strings.Split(s[0], "/")
|
||||
if len(s) < 1 {
|
||||
return
|
||||
}
|
||||
// s[0] is the AccessKeyId
|
||||
accessKeyId := s[0]
|
||||
if accessKeyId == "" {
|
||||
return
|
||||
}
|
||||
// Nil-guard: ensure iam is initialized before lookup
|
||||
if e.iam == nil {
|
||||
glog.V(4).Infof("IAM not initialized, cannot look up access key")
|
||||
return
|
||||
}
|
||||
// Look up the identity by access key to get the username
|
||||
identity, _, found := e.iam.LookupByAccessKey(accessKeyId)
|
||||
if !found {
|
||||
// Mask access key in logs - show only first 4 chars
|
||||
maskedKey := accessKeyId
|
||||
if len(accessKeyId) > 4 {
|
||||
maskedKey = accessKeyId[:4] + "***"
|
||||
}
|
||||
glog.V(4).Infof("Access key %s not found in credential store", maskedKey)
|
||||
return
|
||||
}
|
||||
values.Set("UserName", identity.Name)
|
||||
}
|
||||
|
||||
// iamSelfServiceActions are actions that users can perform on their own resources without admin rights.
|
||||
@@ -2735,27 +2831,79 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
|
||||
}
|
||||
changed = false
|
||||
case "UpdateUser":
|
||||
userName, newUserName := values.Get("UserName"), values.Get("NewUserName")
|
||||
// Taken before the rename: the records the handler changes besides the identity.
|
||||
groups := groupsWithMember(s3cfg, userName)
|
||||
accounts := serviceAccountsOf(s3cfg, userName)
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.UpdateUser(ctx, s3cfg, values)
|
||||
if iamErr != nil {
|
||||
return nil, iamErr
|
||||
}
|
||||
// Write the renamed identity, its groups and service accounts through the credential
|
||||
// manager instead of rewriting every identity via SaveConfiguration. A request that
|
||||
// renames nothing changed nothing, so there is nothing to write either way.
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
renamed := true
|
||||
if newUserName != "" && newUserName != userName {
|
||||
var err error
|
||||
renamed, err = e.persistRenamedIdentity(ctx, s3cfg, userName, newUserName, groups, accounts)
|
||||
if err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
}
|
||||
changed = !renamed
|
||||
}
|
||||
case "DeleteUser":
|
||||
userName := values.Get("UserName")
|
||||
groups := groupsWithMember(s3cfg, userName)
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.DeleteUser(s3cfg, userName)
|
||||
if iamErr != nil {
|
||||
return nil, iamErr
|
||||
}
|
||||
// Targeted delete: one record removed and one removal propagated, plus the groups whose
|
||||
// membership changed, instead of rewriting every remaining identity.
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
if err := e.deleteIdentity(ctx, userName, groups); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "CreateAccessKey":
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.CreateAccessKey(s3cfg, values)
|
||||
userName := values.Get("UserName")
|
||||
resp, iamErr := e.CreateAccessKey(s3cfg, values)
|
||||
if iamErr != nil {
|
||||
glog.Errorf("CreateAccessKey: %+v", iamErr.Error)
|
||||
return nil, iamErr
|
||||
}
|
||||
response = resp
|
||||
// Targeted write: credentialManager.CreateAccessKey adds the key to the one identity
|
||||
// and propagates that identity alone.
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
cred := &iam_pb.Credential{
|
||||
AccessKey: *resp.CreateAccessKeyResult.AccessKey.AccessKeyId,
|
||||
SecretKey: *resp.CreateAccessKeyResult.AccessKey.SecretAccessKey,
|
||||
Status: iamAccessKeyStatusActive,
|
||||
}
|
||||
if err := e.credentialManager.CreateAccessKey(ctx, userName, cred); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "DeleteAccessKey":
|
||||
userName, accessKeyId := values.Get("UserName"), values.Get("AccessKeyId")
|
||||
response = e.DeleteAccessKey(s3cfg, values)
|
||||
// Targeted write: credentialManager.DeleteAccessKey removes the key from the one identity
|
||||
// and propagates that identity alone. The handler answers 200 for a key or user it does
|
||||
// not know, so a not-found from the store is the same no-op it always was.
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
err := e.credentialManager.DeleteAccessKey(ctx, userName, accessKeyId)
|
||||
if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !errors.Is(err, credential.ErrUserNotFound) {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "CreatePolicy":
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.CreatePolicy(ctx, values)
|
||||
@@ -2773,12 +2921,21 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
|
||||
}
|
||||
changed = false
|
||||
case "PutUserPolicy":
|
||||
userName := values.Get("UserName")
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.PutUserPolicy(s3cfg, values)
|
||||
if iamErr != nil {
|
||||
glog.Errorf("PutUserPolicy: %+v", iamErr.Error)
|
||||
return nil, iamErr
|
||||
}
|
||||
// The policy document is already stored; what remains is the identity's recomputed
|
||||
// Actions, written and propagated for that identity alone.
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "GetUserPolicy":
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.GetUserPolicy(s3cfg, values)
|
||||
@@ -2787,11 +2944,18 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
|
||||
}
|
||||
changed = false
|
||||
case "DeleteUserPolicy":
|
||||
userName := values.Get("UserName")
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.DeleteUserPolicy(s3cfg, values)
|
||||
if iamErr != nil {
|
||||
return nil, iamErr
|
||||
}
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "ListUserPolicies":
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.ListUserPolicies(s3cfg, values)
|
||||
@@ -2883,17 +3047,31 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
|
||||
}
|
||||
changed = false
|
||||
case "SetUserStatus":
|
||||
userName := values.Get("UserName")
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.SetUserStatus(s3cfg, values)
|
||||
if iamErr != nil {
|
||||
return nil, iamErr
|
||||
}
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
case "UpdateAccessKey":
|
||||
userName := values.Get("UserName")
|
||||
var iamErr *iamError
|
||||
response, iamErr = e.UpdateAccessKey(s3cfg, values)
|
||||
if iamErr != nil {
|
||||
return nil, iamErr
|
||||
}
|
||||
if e.canPersistIdentity(skipPersist, userName) {
|
||||
if err := e.persistIdentity(ctx, s3cfg, userName); err != nil {
|
||||
return nil, &iamError{Code: CredentialErrToIamErrCode(err), Error: err}
|
||||
}
|
||||
changed = false
|
||||
}
|
||||
// Service Account actions
|
||||
case "CreateServiceAccount":
|
||||
createdBy := values.Get("CreatedBy")
|
||||
@@ -3034,7 +3212,7 @@ func (e *EmbeddedIamApi) ExecuteAction(ctx context.Context, values url.Values, s
|
||||
glog.Errorf("Failed to reload IAM configuration after mutation: %v", err)
|
||||
// Don't fail the request since the persistent save succeeded
|
||||
}
|
||||
} else if action == "AttachUserPolicy" || action == "DetachUserPolicy" || action == "CreatePolicy" || action == "CreatePolicyVersion" || action == "DeletePolicy" || action == "CreateUser" || action == "PutGroupPolicy" || action == "DeleteGroupPolicy" {
|
||||
} else if reloadsAfterTargetedWrite(action) {
|
||||
// Even if changed=false (persisted via credentialManager), we should still reload
|
||||
// if we are utilizing the local in-memory cache for speed
|
||||
if err := e.ReloadConfiguration(); err != nil {
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
package s3api
|
||||
|
||||
// Targeted-write regression tests: single-identity IAM actions must write and
|
||||
// propagate only the identity they changed, not every identity via
|
||||
// SaveConfiguration. See https://github.com/seaweedfs/seaweedfs/issues/11865.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential/memory"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// recordingStore wraps a memory store and counts the write methods the
|
||||
// dispatcher used, so a test can assert the single-identity path was taken and
|
||||
// SaveConfiguration was not. It embeds the concrete store (rather than the
|
||||
// CredentialStore interface) so optional interfaces like InlinePolicyStore and
|
||||
// UserRenamer stay visible through the wrapper.
|
||||
type recordingStore struct {
|
||||
*memory.MemoryStore
|
||||
saves int
|
||||
createAccessKeys int
|
||||
deleteAccessKeys int
|
||||
updateUsers int
|
||||
deleteUsers int
|
||||
updateGroups int
|
||||
}
|
||||
|
||||
func (r *recordingStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
|
||||
r.saves++
|
||||
return r.MemoryStore.SaveConfiguration(ctx, config)
|
||||
}
|
||||
|
||||
func (r *recordingStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error {
|
||||
r.createAccessKeys++
|
||||
return r.MemoryStore.CreateAccessKey(ctx, username, cred)
|
||||
}
|
||||
|
||||
func (r *recordingStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
|
||||
r.deleteAccessKeys++
|
||||
return r.MemoryStore.DeleteAccessKey(ctx, username, accessKey)
|
||||
}
|
||||
|
||||
func (r *recordingStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error {
|
||||
r.updateUsers++
|
||||
return r.MemoryStore.UpdateUser(ctx, username, identity)
|
||||
}
|
||||
|
||||
func (r *recordingStore) DeleteUser(ctx context.Context, username string) error {
|
||||
r.deleteUsers++
|
||||
return r.MemoryStore.DeleteUser(ctx, username)
|
||||
}
|
||||
|
||||
func (r *recordingStore) UpdateGroup(ctx context.Context, group *iam_pb.Group) error {
|
||||
r.updateGroups++
|
||||
return r.MemoryStore.UpdateGroup(ctx, group)
|
||||
}
|
||||
|
||||
// newTargetedWriteTestApi seeds cfg into the store and wraps the store in a
|
||||
// recordingStore. The initial sync save happens before the recorder is
|
||||
// installed so it does not count.
|
||||
func newTargetedWriteTestApi(t *testing.T, cfg *iam_pb.S3ApiConfiguration) (*EmbeddedIamApiForTest, *recordingStore) {
|
||||
t.Helper()
|
||||
api := NewEmbeddedIamApiForTest()
|
||||
api.mockConfig = cfg
|
||||
// Trigger the one-time mockConfig -> store seed through a read action.
|
||||
_, iamErr := api.ExecuteAction(context.Background(), url.Values{"Action": {"ListUsers"}}, false, "")
|
||||
require.Nil(t, iamErr)
|
||||
rec := &recordingStore{MemoryStore: api.credentialManager.Store.(*memory.MemoryStore)}
|
||||
api.credentialManager.Store = rec
|
||||
return api, rec
|
||||
}
|
||||
|
||||
func executeAction(t *testing.T, api *EmbeddedIamApiForTest, vals url.Values) {
|
||||
t.Helper()
|
||||
_, iamErr := api.ExecuteAction(context.Background(), vals, false, "")
|
||||
require.Nil(t, iamErr)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamCreateAccessKeyTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
resp, iamErr := api.ExecuteAction(ctx, url.Values{
|
||||
"Action": {"CreateAccessKey"},
|
||||
"UserName": {"alice"},
|
||||
}, false, "")
|
||||
require.Nil(t, iamErr)
|
||||
require.Equal(t, 0, rec.saves, "CreateAccessKey must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.createAccessKeys)
|
||||
|
||||
out := resp.(*iamCreateAccessKeyResponse)
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ident.Credentials, 1)
|
||||
assert.Equal(t, *out.CreateAccessKeyResult.AccessKey.AccessKeyId, ident.Credentials[0].AccessKey)
|
||||
|
||||
// The new key resolves in the store.
|
||||
byKey, err := api.credentialManager.GetUserByAccessKey(ctx, *out.CreateAccessKeyResult.AccessKey.AccessKeyId)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "alice", byKey.Name)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamDeleteAccessKeyTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{
|
||||
Name: "alice",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake"}},
|
||||
}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"DeleteAccessKey"},
|
||||
"UserName": {"alice"},
|
||||
"AccessKeyId": {"AKIATESTFAKEKEY00001"},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "DeleteAccessKey must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.deleteAccessKeys)
|
||||
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ident.Credentials)
|
||||
_, err = api.credentialManager.GetUserByAccessKey(ctx, "AKIATESTFAKEKEY00001")
|
||||
require.ErrorIs(t, err, credential.ErrAccessKeyNotFound)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamUpdateAccessKeyTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{
|
||||
Name: "alice",
|
||||
Credentials: []*iam_pb.Credential{
|
||||
{AccessKey: "AKIATESTFAKEKEY00001", SecretKey: "testsecretfake", Status: iamAccessKeyStatusActive},
|
||||
},
|
||||
}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"UpdateAccessKey"},
|
||||
"UserName": {"alice"},
|
||||
"AccessKeyId": {"AKIATESTFAKEKEY00001"},
|
||||
"Status": {iamAccessKeyStatusInactive},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "UpdateAccessKey must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, ident.Credentials, 1)
|
||||
assert.Equal(t, iamAccessKeyStatusInactive, ident.Credentials[0].Status)
|
||||
}
|
||||
|
||||
const iamTargetedTestPolicy = `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::bench/*"]}]
|
||||
}`
|
||||
|
||||
func TestEmbeddedIamPutUserPolicyTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"PutUserPolicy"},
|
||||
"UserName": {"alice"},
|
||||
"PolicyName": {"read-bench"},
|
||||
"PolicyDocument": {iamTargetedTestPolicy},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "PutUserPolicy must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, ident.Actions, "identity Actions must be recomputed and persisted")
|
||||
|
||||
doc, err := api.credentialManager.GetUserInlinePolicy(ctx, "alice", "read-bench")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, doc)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamDeleteUserPolicyTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"PutUserPolicy"},
|
||||
"UserName": {"alice"},
|
||||
"PolicyName": {"read-bench"},
|
||||
"PolicyDocument": {iamTargetedTestPolicy},
|
||||
})
|
||||
require.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"DeleteUserPolicy"},
|
||||
"UserName": {"alice"},
|
||||
"PolicyName": {"read-bench"},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "DeleteUserPolicy must not SaveConfiguration")
|
||||
require.Equal(t, 2, rec.updateUsers)
|
||||
|
||||
policies, err := api.credentialManager.ListUserInlinePolicies(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, policies)
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ident.Actions)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamSetUserStatusTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"SetUserStatus"},
|
||||
"UserName": {"alice"},
|
||||
"Status": {iamAccessKeyStatusInactive},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "SetUserStatus must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.updateUsers)
|
||||
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ident.Disabled)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamUpdateUserRenameTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
|
||||
Groups: []*iam_pb.Group{
|
||||
{Name: "devs", Members: []string{"alice", "bob"}},
|
||||
{Name: "ops", Members: []string{"bob"}},
|
||||
},
|
||||
})
|
||||
ctx := context.Background()
|
||||
// Groups in the seed config must also be created in the store directly:
|
||||
// the memory store only surfaces groups written via CreateGroup.
|
||||
require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
|
||||
require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "ops", Members: []string{"bob"}}))
|
||||
require.NoError(t, api.credentialManager.PutUserInlinePolicy(ctx, "alice", "p1", mustParseIamTargetedTestPolicy(t)))
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"UpdateUser"},
|
||||
"UserName": {"alice"},
|
||||
"NewUserName": {"alice-renamed"},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "UpdateUser rename must not SaveConfiguration")
|
||||
|
||||
_, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.ErrorIs(t, err, credential.ErrUserNotFound)
|
||||
_, err = api.credentialManager.GetUser(ctx, "alice-renamed")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Group membership and inline policies follow the rename.
|
||||
g, err := api.credentialManager.GetGroup(ctx, "devs")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"alice-renamed", "bob"}, g.Members)
|
||||
g, err = api.credentialManager.GetGroup(ctx, "ops")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"bob"}, g.Members)
|
||||
|
||||
policies, err := api.credentialManager.ListUserInlinePolicies(ctx, "alice-renamed")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"p1"}, policies)
|
||||
policies, err = api.credentialManager.ListUserInlinePolicies(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, policies)
|
||||
}
|
||||
|
||||
func TestEmbeddedIamDeleteUserTargeted(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}, {Name: "bob"}},
|
||||
Groups: []*iam_pb.Group{{Name: "devs", Members: []string{"alice", "bob"}}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
require.NoError(t, api.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "devs", Members: []string{"alice", "bob"}}))
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"DeleteUser"},
|
||||
"UserName": {"alice"},
|
||||
})
|
||||
require.Equal(t, 0, rec.saves, "DeleteUser must not SaveConfiguration")
|
||||
require.Equal(t, 1, rec.deleteUsers)
|
||||
require.Equal(t, 1, rec.updateGroups)
|
||||
|
||||
_, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.ErrorIs(t, err, credential.ErrUserNotFound)
|
||||
g, err := api.credentialManager.GetGroup(ctx, "devs")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []string{"bob"}, g.Members)
|
||||
_, err = api.credentialManager.GetUser(ctx, "bob")
|
||||
require.NoError(t, err, "other identities must be untouched")
|
||||
}
|
||||
|
||||
// TestEmbeddedIamTargetedSkipPersist: skipPersist must suppress the targeted
|
||||
// write exactly like it suppresses the full save.
|
||||
func TestEmbeddedIamTargetedSkipPersist(t *testing.T) {
|
||||
api, rec := newTargetedWriteTestApi(t, &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{Name: "alice"}},
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
for _, vals := range []url.Values{
|
||||
{"Action": {"CreateAccessKey"}, "UserName": {"alice"}},
|
||||
{"Action": {"SetUserStatus"}, "UserName": {"alice"}, "Status": {iamAccessKeyStatusInactive}},
|
||||
{"Action": {"PutUserPolicy"}, "UserName": {"alice"}, "PolicyName": {"p"}, "PolicyDocument": {iamTargetedTestPolicy}},
|
||||
} {
|
||||
_, iamErr := api.ExecuteAction(ctx, vals, true /*skipPersist*/, "")
|
||||
require.Nil(t, iamErr, "action %v", vals.Get("Action"))
|
||||
}
|
||||
assert.Equal(t, 0, rec.saves)
|
||||
assert.Equal(t, 0, rec.createAccessKeys)
|
||||
assert.Equal(t, 0, rec.updateUsers)
|
||||
|
||||
ident, err := api.credentialManager.GetUser(ctx, "alice")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ident.Credentials)
|
||||
assert.False(t, ident.Disabled)
|
||||
}
|
||||
|
||||
// TestEmbeddedIamStaticIdentityKeepsFullSave: static identities live outside
|
||||
// the store, so a change to one still takes the full-save path.
|
||||
func TestEmbeddedIamStaticIdentityKeepsFullSave(t *testing.T) {
|
||||
api := NewEmbeddedIamApiForTest()
|
||||
api.mockConfig = &iam_pb.S3ApiConfiguration{}
|
||||
api.credentialManager.SetStaticIdentities([]*iam_pb.Identity{{Name: "static-admin"}})
|
||||
rec := &recordingStore{MemoryStore: api.credentialManager.Store.(*memory.MemoryStore)}
|
||||
api.credentialManager.Store = rec
|
||||
|
||||
// Seed via a read action; static identities are merged into the loaded config.
|
||||
_, iamErr := api.ExecuteAction(context.Background(), url.Values{"Action": {"ListUsers"}}, false, "")
|
||||
require.Nil(t, iamErr)
|
||||
rec.saves = 0
|
||||
|
||||
executeAction(t, api, url.Values{
|
||||
"Action": {"SetUserStatus"},
|
||||
"UserName": {"static-admin"},
|
||||
"Status": {iamAccessKeyStatusInactive},
|
||||
})
|
||||
assert.Equal(t, 1, rec.saves, "static identity changes keep the full-save path")
|
||||
assert.Equal(t, 0, rec.updateUsers)
|
||||
}
|
||||
|
||||
func mustParseIamTargetedTestPolicy(t *testing.T) (doc policy_engine.PolicyDocument) {
|
||||
t.Helper()
|
||||
require.NoError(t, json.Unmarshal([]byte(iamTargetedTestPolicy), &doc))
|
||||
return doc
|
||||
}
|
||||
@@ -372,13 +372,13 @@ func TestEmbeddedIamGetUserImplicitUsername(t *testing.T) {
|
||||
t.Fatalf("failed to load iam config: %v", err)
|
||||
}
|
||||
|
||||
// GetUser request with NO UserName, but signed by accessKey.
|
||||
// GetUser request with NO UserName, carrying the identity authentication
|
||||
// verified for accessKey, the way the middleware records it.
|
||||
form := url.Values{}
|
||||
form.Set("Action", "GetUser")
|
||||
req, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+
|
||||
"/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesig")
|
||||
req = req.WithContext(SetIdentityNameInContext(req.Context(), "TestUser"))
|
||||
|
||||
out := iamGetUserResponse{}
|
||||
rr, err := executeEmbeddedIamRequest(api, req, &out)
|
||||
@@ -1578,29 +1578,15 @@ func TestEmbeddedIamDeleteAccessKey(t *testing.T) {
|
||||
assert.Len(t, api.mockConfig.Identities[0].Credentials, 0)
|
||||
}
|
||||
|
||||
// TestEmbeddedIamHandleImplicitUsername tests implicit username extraction from authorization header
|
||||
// TestEmbeddedIamHandleImplicitUsername checks that the implicit user name is
|
||||
// taken from the authenticated identity in the request context, never from
|
||||
// re-parsing the Authorization header.
|
||||
func TestEmbeddedIamHandleImplicitUsername(t *testing.T) {
|
||||
// Create IAM with test credentials - the handleImplicitUsername function now looks
|
||||
// up the username from the credential store based on AccessKeyId
|
||||
// Note: Using obviously fake access keys to avoid secret scanner false positives
|
||||
iam := &IdentityAccessManagement{}
|
||||
testConfig := &iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{
|
||||
{
|
||||
Name: "testuser1",
|
||||
Credentials: []*iam_pb.Credential{
|
||||
{AccessKey: UserAccessKeyPrefix + "TESTFAKEKEY000001", SecretKey: "testsecretfake"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(testConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load test config: %v", err)
|
||||
}
|
||||
embeddedApi := &EmbeddedIamApi{}
|
||||
|
||||
embeddedApi := &EmbeddedIamApi{
|
||||
iam: iam,
|
||||
withIdentity := func(name string) *http.Request {
|
||||
r := &http.Request{Header: http.Header{}}
|
||||
return r.WithContext(SetIdentityNameInContext(r.Context(), name))
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
@@ -1608,14 +1594,14 @@ func TestEmbeddedIamHandleImplicitUsername(t *testing.T) {
|
||||
values url.Values
|
||||
userName string
|
||||
}{
|
||||
// No authorization header - should not set username
|
||||
// No authenticated identity - should not set username
|
||||
{&http.Request{}, url.Values{}, ""},
|
||||
// Valid auth header with known access key - should look up and find "testuser1"
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"},
|
||||
// Malformed auth header (no Credential=) - should not set username
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// Unknown access key - should not set username
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// Authenticated identity in context - sets its name
|
||||
{withIdentity("testuser1"), url.Values{}, "testuser1"},
|
||||
// A client-controlled Authorization header naming another user is ignored
|
||||
{&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=" + UserAccessKeyPrefix + "TESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""},
|
||||
// An explicit UserName is left alone
|
||||
{withIdentity("testuser1"), url.Values{"UserName": {"other"}}, "other"},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
|
||||
@@ -29,11 +29,12 @@ type VendedCredentials struct {
|
||||
Expiration time.Time
|
||||
}
|
||||
|
||||
// VendTableCredentials mints credentials that can read and write exactly one
|
||||
// table's prefix, for a caller the Iceberg catalog has already authenticated
|
||||
// and authorized. Returns nil when no role is configured, which leaves the
|
||||
// catalog telling the client to keep using its own credentials.
|
||||
func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error) {
|
||||
// VendTableCredentials mints credentials that can read exactly one table's
|
||||
// prefix, and write it too unless readOnly, for a caller the Iceberg catalog
|
||||
// has already authenticated and authorized. Returns nil when no role is
|
||||
// configured, which leaves the catalog telling the client to keep using its
|
||||
// own credentials.
|
||||
func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) {
|
||||
if s3a.icebergCredentialRole == "" {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -49,7 +50,7 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc
|
||||
return nil, fmt.Errorf("STS is not configured")
|
||||
}
|
||||
|
||||
policy, err := tablePrefixSessionPolicy(bucket, prefix)
|
||||
policy, err := tablePrefixSessionPolicy(bucket, prefix, readOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -98,8 +99,9 @@ func (s3a *S3ApiServer) stsService() *sts.STSService {
|
||||
}
|
||||
|
||||
// tablePrefixSessionPolicy narrows a session to one table's files plus the
|
||||
// bucket listing that clients need to resolve them.
|
||||
func tablePrefixSessionPolicy(bucket, prefix string) (string, error) {
|
||||
// bucket listing that clients need to resolve them. A read-only session drops
|
||||
// the write verbs: the catalog vends no more authority than the caller holds.
|
||||
func tablePrefixSessionPolicy(bucket, prefix string, readOnly bool) (string, error) {
|
||||
prefix = strings.Trim(prefix, "/")
|
||||
// A resource pattern is matched with wildcards, so a location carrying one
|
||||
// would widen the session to sibling tables. Refuse rather than escape it:
|
||||
@@ -114,12 +116,17 @@ func tablePrefixSessionPolicy(bucket, prefix string) (string, error) {
|
||||
return "", fmt.Errorf("refusing to scope credentials to a whole bucket: s3://%s", bucket)
|
||||
}
|
||||
|
||||
objectActions := []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"}
|
||||
if readOnly {
|
||||
objectActions = []string{"s3:GetObject"}
|
||||
}
|
||||
|
||||
policy := map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []map[string]interface{}{
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"},
|
||||
"Action": objectActions,
|
||||
"Resource": []string{fmt.Sprintf("arn:aws:s3:::%s/%s/*", bucket, prefix)},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
func TestTablePrefixSessionPolicyScopesToTheTable(t *testing.T) {
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
|
||||
if err != nil {
|
||||
t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
|
||||
}
|
||||
@@ -42,13 +42,43 @@ func TestTablePrefixSessionPolicyScopesToTheTable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A caller the catalog would not let commit gets a session that can read the
|
||||
// table's files and nothing more.
|
||||
func TestTablePrefixSessionPolicyReadOnly(t *testing.T) {
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", true)
|
||||
if err != nil {
|
||||
t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
|
||||
}
|
||||
|
||||
var policy struct {
|
||||
Statement []struct {
|
||||
Action []string `json:"Action"`
|
||||
} `json:"Statement"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &policy); err != nil {
|
||||
t.Fatalf("policy is not valid JSON: %v", err)
|
||||
}
|
||||
|
||||
for _, statement := range policy.Statement {
|
||||
for _, action := range statement.Action {
|
||||
switch action {
|
||||
case "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts":
|
||||
t.Errorf("read-only session policy grants %s", action)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := policy.Statement[0].Action; len(got) != 1 || got[0] != "s3:GetObject" {
|
||||
t.Errorf("object actions = %v, want [s3:GetObject]", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A table at the bucket root has no prefix to scope to, and vending would hand
|
||||
// out read and write over every other table in the bucket.
|
||||
func TestTablePrefixSessionPolicyRefusesAWholeBucket(t *testing.T) {
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", ""); err == nil {
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", "", false); err == nil {
|
||||
t.Error("tablePrefixSessionPolicy() error = nil, want a refusal for an empty prefix")
|
||||
}
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", "/"); err == nil {
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", "/", false); err == nil {
|
||||
t.Error("a prefix of only separators was accepted")
|
||||
}
|
||||
}
|
||||
@@ -56,7 +86,7 @@ func TestTablePrefixSessionPolicyRefusesAWholeBucket(t *testing.T) {
|
||||
// GetBucketLocation carries no prefix, so conditioning it would deny location
|
||||
// discovery to every vended credential.
|
||||
func TestTablePrefixSessionPolicyLeavesBucketLocationUnconditioned(t *testing.T) {
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
|
||||
if err != nil {
|
||||
t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
|
||||
}
|
||||
@@ -101,7 +131,7 @@ func TestIcebergSessionNameIsBounded(t *testing.T) {
|
||||
// telling clients to use their own credentials.
|
||||
func TestVendTableCredentialsDisabledWithoutRole(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
credentials, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t")
|
||||
credentials, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t", false)
|
||||
if err != nil {
|
||||
t.Fatalf("VendTableCredentials() error = %v, want nil", err)
|
||||
}
|
||||
@@ -114,7 +144,7 @@ func TestVendTableCredentialsNeedsSTS(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600)
|
||||
|
||||
if _, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t"); err == nil {
|
||||
if _, err := s3a.VendTableCredentials(context.Background(), "admin", "warehouse", "ns/t", false); err == nil {
|
||||
t.Error("VendTableCredentials() error = nil, want an error when STS is not configured")
|
||||
}
|
||||
}
|
||||
@@ -122,7 +152,7 @@ func TestVendTableCredentialsNeedsSTS(t *testing.T) {
|
||||
// Listing is granted on the bucket, so the prefix condition is what keeps a
|
||||
// vended credential from enumerating other tables.
|
||||
func TestTablePrefixSessionPolicyRestrictsListing(t *testing.T) {
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events")
|
||||
raw, err := tablePrefixSessionPolicy("warehouse", "analytics/events", false)
|
||||
if err != nil {
|
||||
t.Fatalf("tablePrefixSessionPolicy() error = %v", err)
|
||||
}
|
||||
@@ -168,11 +198,11 @@ func TestTablePrefixSessionPolicyRestrictsListing(t *testing.T) {
|
||||
// tables, so it is refused instead of vended.
|
||||
func TestTablePrefixSessionPolicyRefusesWildcards(t *testing.T) {
|
||||
for _, prefix := range []string{"analytics/*", "analytics/ev?nts", "*"} {
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", prefix); err == nil {
|
||||
if _, err := tablePrefixSessionPolicy("warehouse", prefix, false); err == nil {
|
||||
t.Errorf("tablePrefixSessionPolicy(%q) error = nil, want a refusal", prefix)
|
||||
}
|
||||
}
|
||||
if _, err := tablePrefixSessionPolicy("ware*house", "analytics/events"); err == nil {
|
||||
if _, err := tablePrefixSessionPolicy("ware*house", "analytics/events", false); err == nil {
|
||||
t.Error("a wildcard in the bucket name was accepted")
|
||||
}
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func TestAnonymousObjectACLVersionedEmptyDirectoryMarker(t *testing.T) {
|
||||
require.Equal(t, "null", rr.Header().Get("x-amz-version-id"))
|
||||
}
|
||||
rr := serveAnonymousRead(s3a, method, "folder", "", nil)
|
||||
require.Equal(t, 404, rr.Code, "the bare prefix is still not an object")
|
||||
require.Equal(t, 403, rr.Code, "the bare prefix is not an object, and the miss stays masked for a non-listable bucket")
|
||||
marker.Extended[s3_constants.ExtAmzAclKey] = []byte(`[]`)
|
||||
rr = serveAnonymousRead(s3a, method, "folder/", "", nil)
|
||||
require.Equal(t, 403, rr.Code, rr.Body.String())
|
||||
@@ -238,8 +238,12 @@ func TestAnonymousObjectACLVersionedDirectoryIdentity(t *testing.T) {
|
||||
f.entries["/buckets/b/folder/.versions"] = &filer_pb.Entry{IsDirectory: true, Extended: map[string][]byte{s3_constants.ExtLatestVersionIdKey: []byte("")}}
|
||||
}
|
||||
// Equivalent filer-path spellings must not recover a different object's data.
|
||||
for _, object := range []string{"folder/", "folder//", "folder\\"} {
|
||||
assertMissing(serveAnonymousRead(s3a, method, object, route.query, headers))
|
||||
assertMissing(serveAnonymousRead(s3a, method, "folder/", route.query, headers))
|
||||
// Spellings the policy's folder/ grant does not cover are
|
||||
// masked: a miss must be indistinguishable from a denied read.
|
||||
for _, object := range []string{"folder//", "folder\\"} {
|
||||
rr := serveAnonymousRead(s3a, method, object, route.query, headers)
|
||||
require.Equal(t, http.StatusForbidden, rr.Code, rr.Body.String())
|
||||
}
|
||||
if kind == "regular file" || kind == "prefix object" || kind == "empty prefix object" {
|
||||
// A slash-only policy must not authorize data belonging to the bare key.
|
||||
|
||||
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
@@ -110,3 +112,72 @@ func (s3a *S3ApiServer) authorizeAnonymousObjectRead(r *http.Request, bucket, ob
|
||||
}
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
// authorizeAnonymousBucketList reports whether an anonymous caller may learn
|
||||
// which keys exist in the bucket: bucket listing via policy or identity, or a
|
||||
// public-read bucket. Anything else masks the miss as AccessDenied.
|
||||
func (s3a *S3ApiServer) authorizeAnonymousBucketList(r *http.Request, bucket string) s3err.ErrorCode {
|
||||
identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity)
|
||||
object := mux.Vars(r)["object"]
|
||||
// The caller's query parameters would otherwise retarget these checks
|
||||
// (?policy resolves to GetBucketPolicy) or satisfy a prefix condition on an
|
||||
// unrelated subtree, so each runs on a request with a fixed shape: a
|
||||
// ListBucket scoped to the probed key, and a plain read of the key itself.
|
||||
// A caller that could learn the key's existence either way — by listing or
|
||||
// by getting it — gets the real miss.
|
||||
listReq := r.Clone(r.Context())
|
||||
listReq.URL = &url.URL{Path: "/" + bucket, RawQuery: url.Values{"prefix": {object}}.Encode()}
|
||||
// versionId must survive: a read of a specific version resolves to
|
||||
// s3:GetObjectVersion, and dropping it would let a GetObject grant answer
|
||||
// for versions the caller may not read.
|
||||
readQuery := url.Values{}
|
||||
if v := r.URL.Query().Get("versionId"); v != "" {
|
||||
readQuery.Set("versionId", v)
|
||||
}
|
||||
readReq := r.Clone(r.Context())
|
||||
readReq.URL = &url.URL{Path: "/" + bucket + "/" + object, RawQuery: readQuery.Encode()}
|
||||
|
||||
principal := buildPrincipalARN(identity, r)
|
||||
policyAllowed := false
|
||||
code, allowed := s3a.checkPolicyWithEntry(listReq, bucket, "", string(s3_constants.ACTION_LIST), principal, nil)
|
||||
if code != s3err.ErrNone {
|
||||
return code
|
||||
}
|
||||
policyAllowed = allowed
|
||||
if !policyAllowed {
|
||||
code, allowed = s3a.checkPolicyWithEntry(readReq, bucket, object, string(s3_constants.ACTION_READ), principal, nil)
|
||||
if code != s3err.ErrNone {
|
||||
return code
|
||||
}
|
||||
policyAllowed = allowed
|
||||
}
|
||||
if s3a.iam.isActionExplicitlyDeniedByApplicablePolicies(listReq, identity, s3_constants.ACTION_LIST, bucket, "") ||
|
||||
s3a.iam.isActionExplicitlyDeniedByApplicablePolicies(readReq, identity, s3_constants.ACTION_READ, bucket, object) {
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
if policyAllowed {
|
||||
return s3err.ErrNone
|
||||
}
|
||||
if identity != nil &&
|
||||
(s3a.iam.VerifyActionPermission(listReq, identity, s3_constants.ACTION_LIST, bucket, "") == s3err.ErrNone ||
|
||||
s3a.iam.VerifyActionPermission(readReq, identity, s3_constants.ACTION_READ, bucket, object) == s3err.ErrNone) {
|
||||
return s3err.ErrNone
|
||||
}
|
||||
if config, code := s3a.getBucketConfig(bucket); code == s3err.ErrNone && config.IsPublicRead {
|
||||
return s3err.ErrNone
|
||||
}
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
// writeAnonymousAwareNotFound keeps missing-bucket and missing-key misses
|
||||
// indistinguishable from denied reads for anonymous callers: a 404 only
|
||||
// reaches the wire when the bucket is anonymously listable. The denial is
|
||||
// refined like Auth's, so a malformed bucket name answers InvalidBucketName.
|
||||
func (s3a *S3ApiServer) writeAnonymousAwareNotFound(w http.ResponseWriter, r *http.Request, bucket string, code s3err.ErrorCode) {
|
||||
if isAnonymousObjectRead(r) && s3a.authorizeAnonymousBucketList(r, bucket) != s3err.ErrNone {
|
||||
identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity)
|
||||
s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, s3err.ErrAccessDenied))
|
||||
return
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, code)
|
||||
}
|
||||
@@ -299,7 +299,9 @@ func TestAnonymousObjectACLConfiguredIdentity(t *testing.T) {
|
||||
}{
|
||||
{"legacy anonymous permission", false, []Action{s3_constants.ACTION_READ}, "", false, 200},
|
||||
{"identity policy allow", false, nil, `{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, false, 200},
|
||||
{"identity deny beats object ACL", true, nil, `{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, false, 403},
|
||||
// The fixture bucket "b" is shorter than S3's three-character minimum,
|
||||
// so a denial for a resolved identity answers InvalidBucketName.
|
||||
{"identity deny beats object ACL", true, nil, `{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}`, false, 400},
|
||||
{"disabled identity cannot grant", false, []Action{s3_constants.ACTION_READ}, "", true, 403},
|
||||
{"public ACL does not require identity", true, nil, "", true, 200},
|
||||
} {
|
||||
|
||||
@@ -181,8 +181,10 @@ func (s3a *S3ApiServer) parseAndValidateRange(w http.ResponseWriter, r *http.Req
|
||||
return 0, totalSize, false, nil
|
||||
}
|
||||
|
||||
// Empty directory: 404. A file promoted to a directory keeps its data and stays retrievable.
|
||||
if entry.IsDirectory && totalSize == 0 {
|
||||
// Empty directory: 404. A file promoted to a directory keeps its data and
|
||||
// stays retrievable, and a stored directory-key marker is an object, not a
|
||||
// missing key — an unsatisfiable range answers 416.
|
||||
if entry.IsDirectory && totalSize == 0 && !entry.IsDirectoryKeyObject() {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return 0, 0, false, newStreamErrorWithResponse(fmt.Errorf("directory object %s/%s cannot be retrieved", bucket, object))
|
||||
}
|
||||
@@ -533,6 +535,24 @@ func (s3a *S3ApiServer) serveDirectoryContent(w http.ResponseWriter, r *http.Req
|
||||
if r.Method == http.MethodHead && len(entry.Chunks) > 0 {
|
||||
contentLength = int64(entry.Attributes.FileSize)
|
||||
}
|
||||
statusCode := http.StatusOK
|
||||
if r.Header.Get("Range") != "" && requestedPartNumber(r) == 0 {
|
||||
// A ranged request answers like the object path does: 206 with its
|
||||
// Content-Range and the range's length, served on GET, reported on HEAD.
|
||||
totalSize := contentLength
|
||||
offset, size, isRangeRequest, rangeErr := s3a.parseAndValidateRange(w, r, entry, totalSize, bucket, object)
|
||||
if rangeErr != nil {
|
||||
return
|
||||
}
|
||||
if isRangeRequest {
|
||||
contentLength = size
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, offset+size-1, totalSize))
|
||||
statusCode = http.StatusPartialContent
|
||||
if r.Method != http.MethodHead {
|
||||
content = content[offset : offset+size]
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
|
||||
|
||||
// Set last modified
|
||||
@@ -545,12 +565,12 @@ func (s3a *S3ApiServer) serveDirectoryContent(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// For HEAD requests, don't write body
|
||||
if r.Method == http.MethodHead {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.WriteHeader(statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Write content
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.WriteHeader(statusCode)
|
||||
if len(content) > 0 {
|
||||
if _, err := w.Write(content); err != nil {
|
||||
glog.Errorf("serveDirectoryContent: failed to write response: %v", err)
|
||||
@@ -663,7 +683,7 @@ func (s3a *S3ApiServer) handleDirectoryObjectRequest(w http.ResponseWriter, r *h
|
||||
} else if isDirectoryObject {
|
||||
// Directory object but doesn't exist
|
||||
glog.V(2).Infof("%s: directory object %s/%s not found", handlerName, bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return true // Request was handled (with not found)
|
||||
}
|
||||
|
||||
@@ -770,7 +790,11 @@ func (s3a *S3ApiServer) processConditionalHeaders(w http.ResponseWriter, r *http
|
||||
w.Header().Set("ETag", result.ETag)
|
||||
}
|
||||
|
||||
s3err.WriteErrorResponse(w, r, result.ErrorCode)
|
||||
if result.ErrorCode == s3err.ErrNoSuchKey || result.ErrorCode == s3err.ErrNoSuchBucket {
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, result.ErrorCode)
|
||||
} else {
|
||||
s3err.WriteErrorResponse(w, r, result.ErrorCode)
|
||||
}
|
||||
return result, true // request handled
|
||||
}
|
||||
return result, false // request not handled
|
||||
@@ -780,7 +804,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
|
||||
if err := s3a.validateTableBucketObjectPath(bucket, object); err != nil {
|
||||
if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, object); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
@@ -841,7 +865,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
versioningConfigured, err = s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
|
||||
@@ -860,7 +884,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get specific version %s: %v", versionId, err)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchVersion)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchVersion)
|
||||
} else {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
}
|
||||
@@ -886,7 +910,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
} else if errors.Is(versionsErr, filer_pb.ErrNotFound) {
|
||||
@@ -900,7 +924,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
} else {
|
||||
// No object at regular path either - object doesn't exist
|
||||
glog.V(3).Infof("GetObject: object not found at regular path or .versions for %s/%s", bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
@@ -909,7 +933,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -930,7 +954,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
// Check if this is a delete marker
|
||||
if entry.Extended != nil {
|
||||
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -975,7 +999,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
if objectEntryForSSE == nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -992,7 +1016,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
// A bare directory path is not an object: 404 like AWS instead of an empty 200,
|
||||
// so Hadoop-style clients fall back to LIST discovery. HEAD and ranged GET already 404.
|
||||
if !strings.HasSuffix(object, "/") && isBareDirectory(objectEntryForSSE) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2475,7 +2499,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object)
|
||||
if err := s3a.validateTableBucketObjectPath(bucket, object); err != nil {
|
||||
if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, object); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
@@ -2518,7 +2542,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
versioningConfigured, err = s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
|
||||
@@ -2536,7 +2560,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get specific version %s for %s/%s: %v", versionId, bucket, object, err)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchVersion)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchVersion)
|
||||
} else {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
}
|
||||
@@ -2562,7 +2586,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("HeadObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
} else if errors.Is(versionsErr, filer_pb.ErrNotFound) {
|
||||
@@ -2576,7 +2600,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
} else {
|
||||
// No object at regular path either - object doesn't exist
|
||||
glog.V(3).Infof("HeadObject: object not found at regular path or .versions for %s/%s", bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
@@ -2585,7 +2609,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("HeadObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -2606,7 +2630,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
// Check if this is a delete marker
|
||||
if entry.Extended != nil {
|
||||
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -2644,7 +2668,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
if objectEntryForSSE == nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -2710,7 +2734,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
isZeroByteFile := objectEntryForSSE.Attributes.FileSize == 0 && !objectEntryForSSE.IsDirectory
|
||||
// A directory with data (a promoted file) is retrievable; empty directories 404 for LIST fallback.
|
||||
if isBareDirectory(objectEntryForSSE) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
if isZeroByteFile {
|
||||
@@ -2718,7 +2742,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
if s3a.hasChildren(r.Context(), bucket, object) {
|
||||
// This is an implicit directory with children
|
||||
// Return 404 to force clients (like s3fs) to use LIST-based discovery
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
s3a.writeAnonymousAwareNotFound(w, r, bucket, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -2741,6 +2765,17 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", startOffset, endOffset, totalSize))
|
||||
statusCode = http.StatusPartialContent
|
||||
glog.V(3).Infof("HeadObject: part %d is bytes %d-%d/%d", partNumber, startOffset, endOffset, totalSize)
|
||||
} else if r.Header.Get("Range") != "" {
|
||||
// A ranged HEAD reports the range like GET does: 206 with its Content-Range and length
|
||||
offset, size, isRangeRequest, rangeErr := s3a.parseAndValidateRange(w, r, objectEntryForSSE, totalSize, bucket, object)
|
||||
if rangeErr != nil {
|
||||
return
|
||||
}
|
||||
if isRangeRequest {
|
||||
responseSize = size
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, offset+size-1, totalSize))
|
||||
statusCode = http.StatusPartialContent
|
||||
}
|
||||
}
|
||||
|
||||
s3a.setResponseHeaders(w, r, objectEntryForSSE, responseSize)
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
// A conditional write with If-Match has nothing to compare against when the key has
|
||||
// no current version; AWS answers 404 there, not 412.
|
||||
func TestValidateConditionalHeadersMissingObject(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
|
||||
existing := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
|
||||
Extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
|
||||
}
|
||||
deleteMarker := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: time.Now().Unix()},
|
||||
Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
header string
|
||||
value string
|
||||
entry *filer_pb.Entry
|
||||
want s3err.ErrorCode
|
||||
}{
|
||||
{"if-match on missing object", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", nil, s3err.ErrNoSuchKey},
|
||||
{"if-match star on missing object", s3_constants.IfMatch, "*", nil, s3err.ErrNoSuchKey},
|
||||
{"if-match on delete marker", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", deleteMarker, s3err.ErrNoSuchKey},
|
||||
{"if-match mismatch on existing object", s3_constants.IfMatch, "0000", existing, s3err.ErrPreconditionFailed},
|
||||
{"if-match hit on existing object", s3_constants.IfMatch, "d41d8cd98f00b204e9800998ecf8427e", existing, s3err.ErrNone},
|
||||
{"if-none-match star on missing object", s3_constants.IfNoneMatch, "*", nil, s3err.ErrNone},
|
||||
{"if-none-match star on delete marker", s3_constants.IfNoneMatch, "*", deleteMarker, s3err.ErrNone},
|
||||
{"if-none-match star on existing object", s3_constants.IfNoneMatch, "*", existing, s3err.ErrPreconditionFailed},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest(http.MethodPut, "/bucket/object", nil)
|
||||
r.Header.Set(tc.header, tc.value)
|
||||
headers, errCode := parseConditionalHeaders(r)
|
||||
if errCode != s3err.ErrNone {
|
||||
t.Fatalf("parseConditionalHeaders: %v", errCode)
|
||||
}
|
||||
if got := s3a.validateConditionalHeaders(r, headers, tc.entry, "bucket", "object"); got != tc.want {
|
||||
t.Errorf("got %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,21 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -1146,7 +1150,9 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
if uploadEntryHasSSE(uploadEntry) || sourceEntryIsEncrypted(entry) || uploadEntryHasChecksum(uploadEntry) {
|
||||
// Inline-Content sources have no chunks for the raw-byte path to copy;
|
||||
// the plaintext stream serves their range instead.
|
||||
if uploadEntryHasSSE(uploadEntry) || sourceEntryIsEncrypted(entry) || uploadEntryHasChecksum(uploadEntry) || len(entry.Content) > 0 {
|
||||
etag, sseMetadata, errCode := s3a.copyObjectPartViaReencryption(r, entry, startOffset, endOffset, dstBucket, dstObject, uploadID, partID, uploadEntry)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
@@ -1204,15 +1210,24 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
// For zero-size files or invalid ranges, create an empty part with size 0
|
||||
dstEntry.Attributes.FileSize = 0
|
||||
dstEntry.Chunks = nil
|
||||
dstEntry.Attributes.Md5 = md5.New().Sum(nil)
|
||||
} else {
|
||||
// Copy chunks that overlap with the range
|
||||
dstChunks, err := s3a.copyChunksForRange(entry, startOffset, endOffset, dstAssignPath)
|
||||
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, startOffset, endOffset, dstAssignPath)
|
||||
if err != nil {
|
||||
glog.Errorf("CopyObjectPartHandler copy chunks error: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3a.mapCopyErrorToS3Error(err))
|
||||
return
|
||||
}
|
||||
dstEntry.Chunks = dstChunks
|
||||
dstEntry.Attributes.Md5 = partMd5
|
||||
}
|
||||
|
||||
// Mirror putToFiler: the part entry carries the MD5 of its bytes so
|
||||
// CopyPartResult, ListParts and complete's composite ETag all describe
|
||||
// the copied range rather than the source chunks.
|
||||
if len(dstEntry.Attributes.Md5) > 0 {
|
||||
dstEntry.Extended[s3_constants.ExtETagKey] = []byte(fmt.Sprintf("%x", dstEntry.Attributes.Md5))
|
||||
}
|
||||
|
||||
// Save the part entry to the multipart uploads folder
|
||||
@@ -1479,7 +1494,7 @@ func (s3a *S3ApiServer) copySingleChunk(chunk *filer_pb.FileChunk, dstPath strin
|
||||
// the source volume for compressed bytes, so a gzipped chunk is
|
||||
// forwarded to the destination without anyone having to decompress.
|
||||
if canStreamCopyChunk(chunk) {
|
||||
if err := s3a.streamCopyChunkRange(context.Background(), srcUrl, fileId, 0, int64(chunk.Size), true /*isFullChunk*/, assignResult); err != nil {
|
||||
if err := s3a.streamCopyChunkRange(context.Background(), srcUrl, fileId, 0, int64(chunk.Size), true /*isFullChunk*/, assignResult, nil); err != nil {
|
||||
return nil, fmt.Errorf("stream chunk: %w", err)
|
||||
}
|
||||
return dstChunk, nil
|
||||
@@ -1534,9 +1549,22 @@ func (s3a *S3ApiServer) copySingleChunkForRange(originalChunk, rangeChunk *filer
|
||||
// part-size = chunk-size assemble pattern hits this branch.
|
||||
if canStreamCopyChunk(originalChunk) {
|
||||
isFullChunk := offsetInChunk == 0 && rangeChunk.Size == originalChunk.Size
|
||||
if err := s3a.streamCopyChunkRange(context.Background(), srcUrl, fileId, offsetInChunk, int64(rangeChunk.Size), isFullChunk, assignResult); err != nil {
|
||||
// A partial range stores different bytes than the source chunk, so
|
||||
// its ETag has to be recomputed; a whole-chunk copy reuses it.
|
||||
var digest hash.Hash
|
||||
if !isFullChunk {
|
||||
digest = md5.New()
|
||||
}
|
||||
if err := s3a.streamCopyChunkRange(context.Background(), srcUrl, fileId, offsetInChunk, int64(rangeChunk.Size), isFullChunk, assignResult, digest); err != nil {
|
||||
return nil, fmt.Errorf("stream chunk range: %w", err)
|
||||
}
|
||||
if digest != nil {
|
||||
dstChunk.ETag = base64.StdEncoding.EncodeToString(digest.Sum(nil))
|
||||
// The range fetch returns raw bytes and the destination stores
|
||||
// them uncompressed, so the source chunk's gzip flag must not
|
||||
// carry over onto a partial copy.
|
||||
dstChunk.IsCompressed = false
|
||||
}
|
||||
return dstChunk, nil
|
||||
}
|
||||
|
||||
@@ -1609,8 +1637,16 @@ func parseRangeHeader(rangeHeader string, fileSize int64) (startOffset, endOffse
|
||||
return startOffset, endOffset, nil
|
||||
}
|
||||
|
||||
// copyChunksForRange copies chunks that overlap with the specified range
|
||||
func (s3a *S3ApiServer) copyChunksForRange(entry *filer_pb.Entry, startOffset, endOffset int64, dstPath string) ([]*filer_pb.FileChunk, error) {
|
||||
// partMd5Result carries the whole-part MD5 the copiedRangeMd5 goroutine
|
||||
// computes beside the per-chunk copies.
|
||||
type partMd5Result struct {
|
||||
sum []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// copyChunksForRange copies chunks that overlap with the specified range and
|
||||
// returns the MD5 of the copied bytes for the part entry's Attributes.Md5.
|
||||
func (s3a *S3ApiServer) copyChunksForRange(entry *filer_pb.Entry, startOffset, endOffset int64, dstPath string) ([]*filer_pb.FileChunk, []byte, error) {
|
||||
var relevantChunks []*filer_pb.FileChunk
|
||||
var originalChunks []*filer_pb.FileChunk
|
||||
|
||||
@@ -1641,6 +1677,36 @@ func (s3a *S3ApiServer) copyChunksForRange(entry *filer_pb.Entry, startOffset, e
|
||||
}
|
||||
}
|
||||
|
||||
// The part's Attributes.Md5 must cover the copied range in byte order,
|
||||
// and md5 cannot combine per-chunk digests. A single chunk's own digest
|
||||
// is the answer only when it supplies the whole range; anything else —
|
||||
// several chunks, or gaps a sparse source zero-fills — needs an ordered
|
||||
// pass the concurrent copies cannot feed, so a second stream of the
|
||||
// source range hashes beside them.
|
||||
partSize := endOffset - startOffset + 1
|
||||
singleChunkCovers := len(relevantChunks) == 1 &&
|
||||
relevantChunks[0].Offset == 0 && int64(relevantChunks[0].Size) == partSize
|
||||
|
||||
md5Ctx, cancelMd5 := context.WithCancel(context.Background())
|
||||
defer cancelMd5()
|
||||
|
||||
var partMd5 []byte
|
||||
var partMd5Ch chan partMd5Result
|
||||
switch {
|
||||
case singleChunkCovers:
|
||||
// resolved below from the copied chunk's own digest
|
||||
case len(entry.GetChunks()) == 0:
|
||||
// no chunks to read at all: the part is partSize zero bytes (the
|
||||
// stream helper returns empty for a chunkless entry, not zero-fill)
|
||||
partMd5 = md5Zeros(partSize)
|
||||
default:
|
||||
partMd5Ch = make(chan partMd5Result, 1)
|
||||
go func() {
|
||||
sum, err := s3a.copiedRangeMd5(md5Ctx, entry, startOffset, endOffset)
|
||||
partMd5Ch <- partMd5Result{sum, err}
|
||||
}()
|
||||
}
|
||||
|
||||
// Copy the relevant chunks using a specialized method for range copies
|
||||
dstChunks := make([]*filer_pb.FileChunk, len(relevantChunks))
|
||||
const defaultChunkCopyConcurrency = 4
|
||||
@@ -1664,11 +1730,79 @@ func (s3a *S3ApiServer) copyChunksForRange(entry *filer_pb.Entry, startOffset, e
|
||||
// Wait for all operations to complete and check for errors
|
||||
for i := 0; i < len(relevantChunks); i++ {
|
||||
if err := <-errChan; err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return dstChunks, nil
|
||||
// Reads resolve overlapping chunks by ModifiedTsNs, and the concurrent
|
||||
// copies stamped arbitrary finish times: rank the destination chunks by
|
||||
// the source's timestamps so the part resolves overlaps the way the
|
||||
// source did (and the way copiedRangeMd5 hashed it).
|
||||
if len(dstChunks) > 1 {
|
||||
order := make([]int, len(dstChunks))
|
||||
for i := range order {
|
||||
order[i] = i
|
||||
}
|
||||
slices.SortStableFunc(order, func(a, b int) int {
|
||||
return cmp.Compare(originalChunks[a].ModifiedTsNs, originalChunks[b].ModifiedTsNs)
|
||||
})
|
||||
now := time.Now().UnixNano()
|
||||
for rank, i := range order {
|
||||
dstChunks[i].ModifiedTsNs = now + int64(rank)
|
||||
}
|
||||
}
|
||||
|
||||
if partMd5Ch != nil {
|
||||
if res := <-partMd5Ch; res.err != nil {
|
||||
return nil, nil, fmt.Errorf("md5 of copied range: %w", res.err)
|
||||
} else {
|
||||
partMd5 = res.sum
|
||||
}
|
||||
} else if singleChunkCovers {
|
||||
if partMd5 = util.Base64Md5ToBytes(dstChunks[0].ETag); len(partMd5) != md5.Size {
|
||||
// TUS and mount writes leave chunks without an ETag; hash the
|
||||
// copied range when the inherited digest is missing or malformed.
|
||||
sum, err := s3a.copiedRangeMd5(md5Ctx, entry, startOffset, endOffset)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("md5 of copied range: %w", err)
|
||||
}
|
||||
partMd5 = sum
|
||||
}
|
||||
}
|
||||
|
||||
return dstChunks, partMd5, nil
|
||||
}
|
||||
|
||||
// md5Zeros hashes size zero bytes without a volume read — a copied range
|
||||
// that overlaps no source chunk assembles to all zeros.
|
||||
func md5Zeros(size int64) []byte {
|
||||
h := md5.New()
|
||||
var zeros [1 << 16]byte
|
||||
for size > 0 {
|
||||
n := min(size, int64(len(zeros)))
|
||||
h.Write(zeros[:n])
|
||||
size -= n
|
||||
}
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
// copiedRangeMd5 streams the source range once and hashes its plaintext —
|
||||
// exactly the bytes the non-SSE part copy assembles on the destination.
|
||||
func (s3a *S3ApiServer) copiedRangeMd5(ctx context.Context, entry *filer_pb.Entry, startOffset, endOffset int64) ([]byte, error) {
|
||||
size := endOffset - startOffset + 1
|
||||
// The copy walks the chunks, so the hash must too: on an entry that also
|
||||
// carries inline Content the stream helper would prefer Content instead.
|
||||
chunked := &filer_pb.Entry{Chunks: entry.GetChunks()}
|
||||
reader, err := s3a.getEncryptedStreamFromVolumes(ctx, chunked, startOffset, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
h := md5.New()
|
||||
if _, err := io.Copy(h, reader); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.Sum(nil), nil
|
||||
}
|
||||
|
||||
// Helper methods for copy operations to avoid code duplication
|
||||
|
||||
@@ -160,7 +160,7 @@ func BenchmarkCopyChunk_Streamed(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err := s3a.streamCopyChunkRange(context.Background(),
|
||||
env.srcSrv.URL, env.assign.FileId, 0, int64(size),
|
||||
true /*isFullChunk*/, env.assign); err != nil {
|
||||
true /*isFullChunk*/, env.assign, nil); err != nil {
|
||||
b.Fatalf("stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
func TestCopyEntryETagPrefersStoredExtendedETag(t *testing.T) {
|
||||
@@ -145,3 +152,349 @@ func TestParseRangeHeaderRejectsUnsatisfiableRange(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A part copied with x-amz-copy-source-range must report the MD5 of the copied
|
||||
// bytes: per destination chunk for a partial-chunk copy, and the whole-part MD5
|
||||
// the part entry's Attributes.Md5 carries for CopyPartResult / ListParts /
|
||||
// complete's composite ETag.
|
||||
func TestCopyChunksForRangeCopiedETags(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
const chunkSize = 4 << 20
|
||||
payload := make([]byte, 2*chunkSize)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i*31 + 7)
|
||||
}
|
||||
b64md5 := func(data []byte) string {
|
||||
sum := md5.Sum(data)
|
||||
return base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
fid0 := fmt.Sprintf("3,%016x%08x", 101, 202)
|
||||
fid1 := fmt.Sprintf("3,%016x%08x", 103, 204)
|
||||
volume.stored[fid0] = payload[:chunkSize]
|
||||
volume.stored[fid1] = payload[chunkSize:]
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "src",
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
FileSize: uint64(len(payload)),
|
||||
},
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: fid0, Offset: 0, Size: chunkSize, ETag: b64md5(payload[:chunkSize])},
|
||||
{FileId: fid1, Offset: chunkSize, Size: chunkSize, ETag: b64md5(payload[chunkSize:])},
|
||||
},
|
||||
}
|
||||
|
||||
// A range spanning the first chunk fully plus 1 MiB of the second.
|
||||
multiEnd := int64(chunkSize + 1<<20 - 1)
|
||||
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, 0, multiEnd, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange: %v", err)
|
||||
}
|
||||
if len(dstChunks) != 2 {
|
||||
t.Fatalf("copyChunksForRange returned %d chunks, want 2", len(dstChunks))
|
||||
}
|
||||
// A whole-chunk copy forwards the stored bytes, so the source ETag stays valid.
|
||||
if got, want := dstChunks[0].ETag, entry.Chunks[0].ETag; got != want {
|
||||
t.Errorf("whole-chunk copy ETag = %q, want source chunk ETag %q", got, want)
|
||||
}
|
||||
// A partial copy stores only the range bytes; its ETag must be their MD5.
|
||||
if got, want := dstChunks[1].ETag, b64md5(payload[chunkSize:multiEnd+1]); got != want {
|
||||
t.Errorf("partial-chunk copy ETag = %q, want copied-bytes MD5 %q", got, want)
|
||||
}
|
||||
// The copied bytes landed on the destination volume (inside the multipart framing).
|
||||
if stored := volume.stored[dstChunks[1].GetFileIdString()]; !bytes.Contains(stored, payload[chunkSize:multiEnd+1]) {
|
||||
t.Error("destination volume does not hold the copied range bytes")
|
||||
}
|
||||
// The whole-part MD5 hashes every copied byte once, in range order.
|
||||
wantPartMd5 := md5.Sum(payload[:multiEnd+1])
|
||||
if !bytes.Equal(partMd5, wantPartMd5[:]) {
|
||||
t.Errorf("part Md5 = %x, want %x", partMd5, wantPartMd5)
|
||||
}
|
||||
partEntry := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{Md5: partMd5},
|
||||
Chunks: dstChunks,
|
||||
}
|
||||
if got, want := copyEntryETag(partEntry), fmt.Sprintf("%x", wantPartMd5); got != want {
|
||||
t.Errorf("copyEntryETag = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// A range inside one chunk: the part's ETag is the MD5 of those bytes.
|
||||
innerChunks, innerMd5, err := s3a.copyChunksForRange(entry, 100, 199, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange inner range: %v", err)
|
||||
}
|
||||
if len(innerChunks) != 1 {
|
||||
t.Fatalf("inner range returned %d chunks, want 1", len(innerChunks))
|
||||
}
|
||||
if got, want := innerChunks[0].ETag, b64md5(payload[100:200]); got != want {
|
||||
t.Errorf("inner-range chunk ETag = %q, want copied-bytes MD5 %q", got, want)
|
||||
}
|
||||
wantInnerMd5 := md5.Sum(payload[100:200])
|
||||
if !bytes.Equal(innerMd5, wantInnerMd5[:]) {
|
||||
t.Errorf("inner-range part Md5 = %x, want %x", innerMd5, wantInnerMd5)
|
||||
}
|
||||
}
|
||||
|
||||
// A range over a sparse source zero-fills the gaps, so the part MD5 has to
|
||||
// hash the logical range — not just the one chunk that overlaps it.
|
||||
func TestCopyChunksForRangeSparseETags(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
fid := fmt.Sprintf("3,%016x%08x", 105, 206)
|
||||
volume.stored[fid] = []byte("abc")
|
||||
sum := md5.Sum([]byte("abc"))
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "sparse",
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
FileSize: 6,
|
||||
},
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: fid, Offset: 3, Size: 3, ETag: base64.StdEncoding.EncodeToString(sum[:])},
|
||||
},
|
||||
}
|
||||
|
||||
// bytes 0-5 read as "\x00\x00\x00abc"; the part MD5 must cover the zeros too.
|
||||
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, 0, 5, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange sparse: %v", err)
|
||||
}
|
||||
if len(dstChunks) != 1 || dstChunks[0].Offset != 3 {
|
||||
t.Fatalf("sparse copy chunks = %+v, want one chunk at offset 3", dstChunks)
|
||||
}
|
||||
wantMd5 := md5.Sum([]byte("\x00\x00\x00abc"))
|
||||
if !bytes.Equal(partMd5, wantMd5[:]) {
|
||||
t.Errorf("sparse part Md5 = %x, want %x (zero-filled range)", partMd5, wantMd5)
|
||||
}
|
||||
|
||||
// A range over a pure hole copies nothing but still reads as zeros.
|
||||
holeChunks, holeMd5, err := s3a.copyChunksForRange(entry, 0, 2, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange hole: %v", err)
|
||||
}
|
||||
if len(holeChunks) != 0 {
|
||||
t.Fatalf("hole copy returned %d chunks, want 0", len(holeChunks))
|
||||
}
|
||||
wantHoleMd5 := md5.Sum([]byte{0, 0, 0})
|
||||
if !bytes.Equal(holeMd5, wantHoleMd5[:]) {
|
||||
t.Errorf("hole part Md5 = %x, want %x (all zeros)", holeMd5, wantHoleMd5)
|
||||
}
|
||||
}
|
||||
|
||||
// Destination chunks must resolve overlaps the way the source did: reads
|
||||
// pick winners by ModifiedTsNs, so the copied chunks keep the source's
|
||||
// precedence even though they were uploaded concurrently.
|
||||
func TestCopyChunksForRangePreservesSourcePrecedence(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
etagOf := func(b []byte) string {
|
||||
sum := md5.Sum(b)
|
||||
return base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
fidA := fmt.Sprintf("3,%016x%08x", 107, 208)
|
||||
fidB := fmt.Sprintf("3,%016x%08x", 109, 210)
|
||||
volume.stored[fidA] = []byte("aaaa")
|
||||
volume.stored[fidB] = []byte("bbbb")
|
||||
|
||||
// A covers [0,4), B covers [2,6); B's range is copied into a chunk that
|
||||
// overlaps A's copy on [2,4), and the source says A wins there.
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "overlap",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 6},
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: fidA, Offset: 0, Size: 4, ModifiedTsNs: 20, ETag: etagOf([]byte("aaaa"))},
|
||||
{FileId: fidB, Offset: 2, Size: 4, ModifiedTsNs: 10, ETag: etagOf([]byte("bbbb"))},
|
||||
},
|
||||
}
|
||||
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, 0, 5, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange overlap: %v", err)
|
||||
}
|
||||
if len(dstChunks) != 2 {
|
||||
t.Fatalf("overlap copy returned %d chunks, want 2", len(dstChunks))
|
||||
}
|
||||
if dstChunks[0].ModifiedTsNs <= dstChunks[1].ModifiedTsNs {
|
||||
t.Errorf("destination precedence inverted: chunk from ts=20 got %d, chunk from ts=10 got %d",
|
||||
dstChunks[0].ModifiedTsNs, dstChunks[1].ModifiedTsNs)
|
||||
}
|
||||
// The source resolves to "aaaa"+"bb"; the part MD5 must match that view.
|
||||
wantMd5 := md5.Sum([]byte("aaaabb"))
|
||||
if !bytes.Equal(partMd5, wantMd5[:]) {
|
||||
t.Errorf("overlap part Md5 = %x, want %x", partMd5, wantMd5)
|
||||
}
|
||||
}
|
||||
|
||||
// An inline-Content source has no chunks for the raw-byte copy; the plaintext
|
||||
// stream path must write the range's real bytes and tag the part with their
|
||||
// MD5, not a zero-fill digest.
|
||||
func TestCopyObjectPartInlineContent(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
content := []byte("inline-content")
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "src",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(content))},
|
||||
Content: content,
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dst?partNumber=1&uploadId=u1", nil)
|
||||
etag, _, code := s3a.copyObjectPartViaReencryption(r, entry, 2, 5, "b", "dst", "u1", 1, &filer_pb.Entry{})
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("copyObjectPartViaReencryption: %v", code)
|
||||
}
|
||||
wantMd5 := md5.Sum(content[2:6])
|
||||
if got := strings.Trim(etag, `"`); got != fmt.Sprintf("%x", wantMd5) {
|
||||
t.Errorf("etag = %q, want %x", got, wantMd5)
|
||||
}
|
||||
var partEntry *filer_pb.Entry
|
||||
for key, e := range filerImpl.entries {
|
||||
if strings.Contains(key, "u1") {
|
||||
partEntry = e
|
||||
}
|
||||
}
|
||||
if partEntry == nil || len(partEntry.Chunks) == 0 {
|
||||
t.Fatalf("no part entry with chunks stored: %+v", filerImpl.entries)
|
||||
}
|
||||
found := false
|
||||
for _, stored := range volume.stored {
|
||||
if bytes.Contains(stored, content[2:6]) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("destination volume does not hold the copied content bytes")
|
||||
}
|
||||
}
|
||||
|
||||
// The handler must route an inline-Content source away from the raw chunk
|
||||
// copy — CopyObjectPartHandler has to send it through the plaintext stream
|
||||
// path or the part lands as zeros.
|
||||
func TestCopyObjectPartHandlerInlineContent(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
content := []byte("inline-content")
|
||||
uploadID := s3a.generateUploadID("dst")
|
||||
filerImpl.entries["/buckets/b"] = &filer_pb.Entry{Name: "b", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
|
||||
filerImpl.entries["/buckets/b/src"] = &filer_pb.Entry{
|
||||
Name: "src",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(content))},
|
||||
Content: content,
|
||||
}
|
||||
filerImpl.entries["/buckets/b/.uploads/"+uploadID] = &filer_pb.Entry{
|
||||
Name: uploadID,
|
||||
IsDirectory: true,
|
||||
Extended: map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte("dst")},
|
||||
}
|
||||
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dst?partNumber=1&uploadId="+uploadID, nil)
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dst"})
|
||||
r.Header.Set("X-Amz-Copy-Source", "/b/src")
|
||||
r.Header.Set("x-amz-copy-source-range", "bytes=2-5")
|
||||
w := httptest.NewRecorder()
|
||||
s3a.CopyObjectPartHandler(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObjectPartHandler = %d %s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
wantMd5 := fmt.Sprintf("%x", md5.Sum(content[2:6]))
|
||||
if !strings.Contains(w.Body.String(), wantMd5) {
|
||||
t.Errorf("response %q does not contain copied-range ETag %q", w.Body.String(), wantMd5)
|
||||
}
|
||||
var partEntry *filer_pb.Entry
|
||||
for key, e := range filerImpl.entries {
|
||||
if strings.Contains(key, uploadID) && !e.IsDirectory {
|
||||
partEntry = e
|
||||
}
|
||||
}
|
||||
if partEntry == nil || len(partEntry.Chunks) == 0 {
|
||||
t.Fatalf("no part entry with chunks stored: %+v", filerImpl.entries)
|
||||
}
|
||||
found := false
|
||||
for _, stored := range volume.stored {
|
||||
if bytes.Contains(stored, content[2:6]) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("destination volume does not hold the copied content bytes")
|
||||
}
|
||||
}
|
||||
|
||||
// An empty part must carry the empty-body MD5 on the stored entry too, so
|
||||
// ListParts and complete's composite ETag see the same digest CopyPartResult
|
||||
// reported.
|
||||
func TestWriteEmptyCopyPartStoresMd5(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
etag, code := s3a.writeEmptyCopyPart("b", "u1", 3)
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("writeEmptyCopyPart: %v", code)
|
||||
}
|
||||
const emptyMD5Hex = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
if etag != emptyMD5Hex {
|
||||
t.Fatalf("writeEmptyCopyPart etag = %q, want %q", etag, emptyMD5Hex)
|
||||
}
|
||||
var partEntry *filer_pb.Entry
|
||||
for key, e := range filerImpl.entries {
|
||||
if strings.Contains(key, "u1") {
|
||||
partEntry = e
|
||||
}
|
||||
}
|
||||
if partEntry == nil {
|
||||
t.Fatalf("no part entry stored: %+v", filerImpl.entries)
|
||||
}
|
||||
if got := string(partEntry.Extended[s3_constants.ExtETagKey]); got != emptyMD5Hex {
|
||||
t.Errorf("stored ExtETagKey = %q, want %q", got, emptyMD5Hex)
|
||||
}
|
||||
if got := fmt.Sprintf("%x", partEntry.Attributes.Md5); got != emptyMD5Hex {
|
||||
t.Errorf("stored Attributes.Md5 = %q, want %q", got, emptyMD5Hex)
|
||||
}
|
||||
}
|
||||
|
||||
// A whole-range chunk copy normally reuses the source chunk's ETag as the
|
||||
// part MD5; TUS and mount writes leave chunks without one, so the part has
|
||||
// to hash the copied bytes instead of saving an empty digest.
|
||||
func TestCopyChunksForRangeMissingChunkETag(t *testing.T) {
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
fid := fmt.Sprintf("3,%016x%08x", 111, 212)
|
||||
volume.stored[fid] = []byte("etagless")
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "src",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 8},
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{FileId: fid, Offset: 0, Size: 8},
|
||||
},
|
||||
}
|
||||
|
||||
dstChunks, partMd5, err := s3a.copyChunksForRange(entry, 0, 7, "/dst")
|
||||
if err != nil {
|
||||
t.Fatalf("copyChunksForRange: %v", err)
|
||||
}
|
||||
if len(dstChunks) != 1 {
|
||||
t.Fatalf("copyChunksForRange returned %d chunks, want 1", len(dstChunks))
|
||||
}
|
||||
wantMd5 := md5.Sum([]byte("etagless"))
|
||||
if !bytes.Equal(partMd5, wantMd5[:]) {
|
||||
t.Errorf("part Md5 = %x, want %x (hashed copied range)", partMd5, wantMd5)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 186 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user