mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
Chris Luanddevin-ai-integration[bot]
811b8b5734
* add a per-mount cache_wait_ms to the remote storage mount mapping A read of an uncached remote-only object waits on a hardcoded size tier before it can fall back to the origin, so every ranged read of a large remote-only object pays that wait. Carry the wait in the mount mapping so it can be tuned, or set to zero, per mount. * resolve the cache wait of an uncached remote-only read from its mount The wait came only from the object size, so an operator could not trade cache hits for time to first byte. Both read paths now resolve the mount covering the object and let its cache_wait_ms replace the size tiers. * read straight from the remote when a mount waits zero for its cache A mount used as a streaming source pays the cache wait on every ranged read of an object too large to finish caching, and the caching itself is wasted work. A zero wait now skips the cache call, so both read paths go to the origin immediately. * let remote.mount set the cache wait of a mount remote.mount -cacheWait=0 turns a mount into a streaming source, and any other duration trades cache hits against time to first byte. * keep the size based wait for a version-specific read A read pinned to a version cannot fall back to the origin, since the mounted remote only holds the current key, so a mount that opts out of caching would leave it on the 503 retry loop forever. * let the operator allow a remote-only read to dial an internal endpoint The remote-mount read paths in the filer and the S3 gateway always refused an endpoint resolving to a loopback or private host, so a mount backed by an internal S3 could never be read from its origin, only through the local cache. Both now take the allowance the volume server already has, still off by default. * skip the background cache of a mount that waits zero for its cache GetObjectHandler kicks off caching for every remote-only read, so a mount serving as a streaming source kept downloading whole objects even though no read ever waited for them. * cover a zero cache wait end to end The read has to reach a real origin, so the harness also opts the filer and the S3 gateway into dialing the loopback remote it already allows for the volume server. * resolve the S3 cache wait once so the background cache follows it too The background cache that GetObjectHandler starts read the mount on its own, so it skipped a version-specific read that the foreground path still waits for. Both now ask the same resolver. * answer 404 when the origin of a zero-wait read is gone Metadata can outlive the object it points at, and with no cache to fill the read would sit on the 503 retry path forever. The remote backends already report a missing object as ErrRemoteObjectNotFound. * open the origin at write time for a multipart range Every part of a multipart Range is prepared before any is written, so opening eagerly would hold one origin connection per part and leak the ones already opened when a later part fails to open. * reject a cache wait shorter than a millisecond The mapping stores milliseconds, so -cacheWait=500us truncated to zero and silently turned caching off instead of waiting. * restore the doc comment of cacheRemoteObjectForStreamingWithShortTimeout Extracting the wait resolver left its comment on the new function. * stat the origin before committing a multipart range Opening at write time keeps no connection through the preparation, but it also moved a failure past the point where the multipart body picks the response status, so a gone origin truncated a 206 instead of answering 404. One stat up front puts the status back. * stat the origin once per request Every part of a multipart Range is prepared on its own, so the preflight ran once per range instead of once per read. * map Azure and GCS stream not-found to ErrRemoteObjectNotFound ReadFileAsStream on Azure and GCS returned provider-specific not-found errors instead of ErrRemoteObjectNotFound, so a zero-wait read of a deleted object was misclassified as a transient cache failure and retried indefinitely. Map BlobNotFound and ErrObjectNotExist the same way StatFile already does. * Update weed/remote_storage/gcs/gcs_storage_client.go Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
81 lines
2.1 KiB
Protocol Buffer
81 lines
2.1 KiB
Protocol Buffer
syntax = "proto3";
|
|
|
|
package remote_pb;
|
|
|
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb";
|
|
option java_package = "seaweedfs.client";
|
|
option java_outer_classname = "FilerProto";
|
|
|
|
/////////////////////////
|
|
// Remote Storage related
|
|
/////////////////////////
|
|
message RemoteConf {
|
|
string type = 1;
|
|
string name = 2;
|
|
string s3_access_key = 4;
|
|
string s3_secret_key = 5;
|
|
string s3_region = 6;
|
|
string s3_endpoint = 7;
|
|
string s3_storage_class = 8;
|
|
bool s3_force_path_style = 9;
|
|
bool s3_support_tagging = 13;
|
|
bool s3_v4_signature = 11;
|
|
|
|
string gcs_google_application_credentials = 10;
|
|
string gcs_project_id = 12;
|
|
|
|
string azure_account_name = 15;
|
|
string azure_account_key = 16;
|
|
string azure_client_id = 17;
|
|
string azure_endpoint = 18;
|
|
|
|
string backblaze_key_id = 20;
|
|
string backblaze_application_key = 21;
|
|
string backblaze_endpoint = 22;
|
|
string backblaze_region = 23;
|
|
|
|
string aliyun_access_key = 25;
|
|
string aliyun_secret_key = 26;
|
|
string aliyun_endpoint = 27;
|
|
string aliyun_region = 28;
|
|
|
|
string tencent_secret_id = 30;
|
|
string tencent_secret_key = 31;
|
|
string tencent_endpoint = 32;
|
|
|
|
string baidu_access_key = 35;
|
|
string baidu_secret_key = 36;
|
|
string baidu_endpoint = 37;
|
|
string baidu_region = 38;
|
|
|
|
string wasabi_access_key = 40;
|
|
string wasabi_secret_key = 41;
|
|
string wasabi_endpoint = 42;
|
|
string wasabi_region = 43;
|
|
|
|
string filebase_access_key = 60;
|
|
string filebase_secret_key = 61;
|
|
string filebase_endpoint = 62;
|
|
|
|
string storj_access_key = 65;
|
|
string storj_secret_key = 66;
|
|
string storj_endpoint = 67;
|
|
|
|
string contabo_access_key = 68;
|
|
string contabo_secret_key = 69;
|
|
string contabo_endpoint = 70;
|
|
string contabo_region = 71;
|
|
}
|
|
|
|
message RemoteStorageMapping {
|
|
map<string,RemoteStorageLocation> mappings = 1;
|
|
string primary_bucket_storage_name = 2;
|
|
}
|
|
message RemoteStorageLocation {
|
|
string name = 1;
|
|
string bucket = 2;
|
|
string path = 3;
|
|
int32 listing_cache_ttl_seconds = 4; // 0 = disabled; >0 enables on-demand directory listing with this TTL in seconds
|
|
optional int32 cache_wait_ms = 5; // unset = size based default; 0 = read straight from the remote without caching
|
|
}
|