seaweedfs/weed/pb/volume_server.proto

syntax = "proto3";

package volume_server_pb;
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";

import "remote.proto";

//////////////////////////////////////////////////

// Persistent state for volume servers.
message VolumeServerState {
    // whether the server is in maintenance (i.e. read-only) mode.
    bool maintenance = 1;
    // incremental version counter
    uint32 version = 2;
}

//////////////////////////////////////////////////

service VolumeServer {
    //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
    rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
    }

    rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
    }
    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
    }
    rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
    }
    rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
    }

    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
    }
    rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
    }

    rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
    }
    rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
    }

    rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
    }
    rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
    }
    rpc VolumeConsolidateIndex (VolumeConsolidateIndexRequest) returns (VolumeConsolidateIndexResponse) {
    }
    rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
    }
    rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
    }
    rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
    }
    rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
    }
    rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
    }

    rpc GetState (GetStateRequest) returns (GetStateResponse) {
    }
    rpc SetState (SetStateRequest) returns (SetStateResponse) {
    }

    // copy the .idx .dat files, and mount this volume
    rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
    }
    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
    }
    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
    }
    rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) {
    }

    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
    }
    rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
    }
    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
    }
    rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
    }

    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
    }
    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
    }

    // erasure coding
    rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
    }
    rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
    }
    rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
    }
    rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
    }
    rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
    }
    rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
    }
    rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
    }
    rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
    }
    rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
    }
    rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) {
    }

    // tiered storage
    rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
    }
    rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
    }

    rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
    }
    rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
    }

    // remote storage
    rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
    }

    // scrubbing
    rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
    }
    rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
    }

    // <experimental> query
    rpc Query (QueryRequest) returns (stream QueriedStripe) {
    }

    rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
    }

    rpc Ping (PingRequest) returns (PingResponse) {
    }

}

//////////////////////////////////////////////////

message BatchDeleteRequest {
    repeated string file_ids = 1;
    bool skip_cookie_check = 2;
}

message BatchDeleteResponse {
    repeated DeleteResult results = 1;
}
message DeleteResult {
    string file_id = 1;
    int32 status = 2;
    string error = 3;
    uint32 size = 4;
    uint32 version = 5;
}

message Empty {
}

message VacuumVolumeCheckRequest {
    uint32 volume_id = 1;
}
message VacuumVolumeCheckResponse {
    double garbage_ratio = 1;
    bool disk_space_low = 4; // the volume is read-only solely because its disk is low on space — a cause compaction itself reclaims
}

message VacuumVolumeCompactRequest {
    uint32 volume_id = 1;
    int64 preallocate = 2;
}
message VacuumVolumeCompactResponse {
    int64 processed_bytes = 1;
    float load_avg_1m = 2;
}

message VacuumVolumeCommitRequest {
    uint32 volume_id = 1;
}
message VacuumVolumeCommitResponse {
    bool is_read_only = 1;
    uint64 volume_size = 2;
}

message VacuumVolumeCleanupRequest {
    uint32 volume_id = 1;
}
message VacuumVolumeCleanupResponse {
}

message DeleteCollectionRequest {
    string collection = 1;
}
message DeleteCollectionResponse {
}

message AllocateVolumeRequest {
    uint32 volume_id = 1;
    string collection = 2;
    int64 preallocate = 3;
    string replication = 4;
    string ttl = 5;
    uint32 memory_map_max_size_mb = 6;
    string disk_type = 7;
    uint32 version = 8;
}
message AllocateVolumeResponse {
}

message VolumeSyncStatusRequest {
    uint32 volume_id = 1;
}
message VolumeSyncStatusResponse {
    uint32 volume_id = 1;
    string collection = 2;
    string replication = 4;
    string ttl = 5;
    uint64 tail_offset = 6;
    uint32 compact_revision = 7;
    uint64 idx_file_size = 8;
    uint32 version = 9;
}

message VolumeIncrementalCopyRequest {
    uint32 volume_id = 1;
    uint64 since_ns = 2;
    // The backup's index position and last entry. When set, the replica
    // resumes by entry position instead of by its own append_at_ns stamp,
    // which differs across replicas for the same needle.
    uint64 entry_count = 3;
    uint64 last_needle_key = 4;
    uint32 last_needle_cookie = 5;
    int32 last_needle_size = 6;
    // FNV-1a checksum of the (key, size) of every local index entry,
    // verifying the whole prefix since replica record order can differ.
    uint64 entries_hash = 7;
}
message VolumeIncrementalCopyResponse {
    bytes file_content = 1;
    // Index rows after the copy's entry_count, verbatim. A remote-tier
    // tombstone is an index-only row with no .dat record, so it can only
    // reach the backup through this stream.
    bytes idx_content = 2;
}

message VolumeMountRequest {
    uint32 volume_id = 1;
    optional string collection = 2;
}
message VolumeMountResponse {
}

message VolumeUnmountRequest {
    uint32 volume_id = 1;
}
message VolumeUnmountResponse {
}

message VolumeConsolidateIndexRequest {
    uint32 volume_id = 1;
}
message VolumeConsolidateIndexResponse {
}

message VolumeDeleteRequest {
    uint32 volume_id = 1;
    bool only_empty = 2;
    // when true, do not remove the cloud-tier object backing the volume.
    // used for moves where another server is taking over the same .vif.
    bool keep_remote_data = 3;
    // when true, delete only if every needle is deleted: the volume held
    // data once but nothing is live anymore. Passing either check,
    // only_empty or this one, is enough to delete.
    bool only_garbage = 4;
}
message VolumeDeleteResponse {
}

message VolumeMarkReadonlyRequest {
    uint32 volume_id = 1;
    bool persist = 2;
    // reject writes but keep accepting deletes, so expiring data can drain the volume
    bool can_delete = 3;
}
message VolumeMarkReadonlyResponse {
}

message VolumeMarkWritableRequest {
    uint32 volume_id = 1;
}
message VolumeMarkWritableResponse {
}

message VolumeConfigureRequest {
    uint32 volume_id = 1;
    string replication = 2;
}
message VolumeConfigureResponse {
    string error = 1;
}

message VolumeStatusRequest {
    uint32 volume_id = 1;
}
message VolumeStatusResponse {
    bool is_read_only = 1;
    uint64 volume_size = 2;
    uint64 file_count = 3;
    uint64 file_deleted_count = 4;
}

message GetStateRequest {
}
message GetStateResponse {
    VolumeServerState state = 1;
}

message SetStateRequest {
    // SetState updates *all* volume server flags at once. Retrieve state with GetState(),
    // modify individual flags as required, then call this RPC to update.
    VolumeServerState state = 1;
}
message SetStateResponse {
    VolumeServerState state = 1;
}

message VolumeCopyRequest {
    uint32 volume_id = 1;
    string collection = 2;
    string replication = 3;
    string ttl = 4;
    string source_data_node = 5;
    string disk_type = 6;
    int64 io_byte_per_second = 7;
}
message VolumeCopyResponse {
    uint64 last_append_at_ns = 1;
    int64 processed_bytes = 2;
}

message CopyFileRequest {
    uint32 volume_id = 1;
    string ext = 2;
    uint32 compaction_revision = 3;
    uint64 stop_offset = 4;
    string collection = 5;
    bool is_ec_volume = 6;
    bool ignore_source_file_not_found = 7;
}
message CopyFileResponse {
    bytes file_content = 1;
    int64 modified_ts_ns = 2;
}

message ReceiveFileRequest {
    oneof data {
        ReceiveFileInfo info = 1;
        bytes file_content = 2;
    }
}

message ReceiveFileInfo {
    uint32 volume_id = 1;
    string ext = 2;
    string collection = 3;
    bool is_ec_volume = 4;
    uint32 shard_id = 5;
    uint64 file_size = 6;
    uint32 disk_id = 7;  // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)

    // Field numbers 8-11 are reserved for versioned-EC; disk_type stays at 12.
    // Staged-new-volume mode (EC decode onto a clean peer): set on a non-EC push
    // whose volume does not yet exist on this server. The server picks a disk
    // location of this medium with a free slot and writes <base><ext>.copying,
    // finalized by VolumeEcShardsToVolume(from_staged).
    string disk_type = 12;
}

message ReceiveFileResponse {
    uint64 bytes_written = 1;
    string error = 2;
}

message ReadNeedleBlobRequest {
    uint32 volume_id = 1;
    int64 offset = 3; // actual offset
    int32 size = 4;
}
message ReadNeedleBlobResponse {
    bytes needle_blob = 1;
}

message ReadNeedleMetaRequest {
    uint32 volume_id = 1;
    uint64 needle_id = 2;
    int64 offset = 3; // actual offset
    int32 size = 4;
}
message ReadNeedleMetaResponse {
    uint32 cookie = 1;
    uint64 last_modified = 2;
    uint32 crc = 3;
    string ttl = 4;
    uint64 append_at_ns = 5;
}

message WriteNeedleBlobRequest {
    uint32 volume_id = 1;
    uint64 needle_id = 2;
    int32 size = 3;
    bytes needle_blob = 4;
}
message WriteNeedleBlobResponse {
}

message ReadAllNeedlesRequest {
    repeated uint32 volume_ids = 1;
}
message ReadAllNeedlesResponse {
    uint32 volume_id = 1;
    uint64 needle_id = 2;
    uint32 cookie = 3;
    bytes needle_blob = 5;
    bool needle_blob_compressed = 6;
    uint64 last_modified = 7;
    uint32 crc = 8;
    bytes name = 9;
    bytes mime = 10;
}

message VolumeTailSenderRequest {
    uint32 volume_id = 1;
    uint64 since_ns = 2;
    uint32 idle_timeout_seconds = 3;
}
message VolumeTailSenderResponse {
    bytes needle_header = 1;
    bytes needle_body = 2;
    bool is_last_chunk = 3;
    uint32 version = 4;
}

message VolumeTailReceiverRequest {
    uint32 volume_id = 1;
    uint64 since_ns = 2;
    uint32 idle_timeout_seconds = 3;
    string source_volume_server = 4;
}
message VolumeTailReceiverResponse {
}

message VolumeEcShardsGenerateRequest {
    uint32 volume_id = 1;
    string collection = 2;
}
message VolumeEcShardsGenerateResponse {
}

message VolumeEcShardsRebuildRequest {
    uint32 volume_id = 1;
    string collection = 2;
    bool unsafe_ignore_sidecar = 3; // bypass the bitrot-sidecar fail-closed guard (operator override; distinct from ec.rebuild -force)
}
message VolumeEcShardsRebuildResponse {
    repeated uint32 rebuilt_shard_ids = 1;
}

message VolumeEcShardsCopyRequest {
    uint32 volume_id = 1;
    string collection = 2;
    repeated uint32 shard_ids = 3;
    bool copy_ecx_file = 4;
    string source_data_node = 5;
    bool copy_ecj_file = 6;
    bool copy_vif_file = 7;
    uint32 disk_id = 8;  // Target disk ID for storing EC shards
    bool copy_ecsum_file = 9; // copy the bitrot checksum sidecar (.ecsum) when present; tolerant of a missing source (no-op), since this non-2PC path has no Prepare backstop
    int64 io_byte_per_second = 10; // limit the copy rate; 0 falls back to the server's maintenance rate
}
message VolumeEcShardsCopyResponse {
}

message VolumeEcShardsDeleteRequest {
    uint32 volume_id = 1;
    string collection = 2;
    repeated uint32 shard_ids = 3;
    bool full_teardown = 4; // pre-encode cleanup: wipe every EC artifact + generation for this volume, not just shard_ids
    int64 encode_ts_ns = 5; // full_teardown generation fence: delete only a disk whose .vif generation is strictly OLDER than this; preserve same-or-newer, generation 0, and an unreadable .vif. 0 => wipe-all (shell pre-encode / pre-upgrade)
    uint32 delete_generations_older_than = 6; // post-commit cleanup: delete only staged <base>.*.v<N> artifacts with N strictly below this; 0 disables
}
message VolumeEcShardsDeleteResponse {
    bool full_teardown_done = 1; // set by a new server that performed full_teardown; absent from an old server lets the caller detect the silent no-op
}

message VolumeEcShardsMountRequest {
    uint32 volume_id = 1;
    string collection = 2;
    repeated uint32 shard_ids = 3;
    string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
    bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
}
message VolumeEcShardsMountResponse {
}

message VolumeEcShardsUnmountRequest {
    uint32 volume_id = 1;
    repeated uint32 shard_ids = 3;
    int64 encode_ts_ns = 4; // generation fence: skip a disk whose mounted EC volume is this generation or newer (0 = unfenced, unmount all)
}
message VolumeEcShardsUnmountResponse {
}

message VolumeEcShardReadRequest {
    uint32 volume_id = 1;
    uint32 shard_id = 2;
    int64 offset = 3;
    int64 size = 4;
    uint64 file_key = 5;
    reserved 6;
    int64 encode_ts_ns = 7; // caller's expected encode time; the server rejects a shard from a different encode run
}
message VolumeEcShardReadResponse {
    bytes data = 1;
    bool is_deleted = 2;
    int64 encode_ts_ns = 3; // identity of the shard actually served; client rejects a mismatch (0 = pre-upgrade server)
}

message VolumeEcBlobDeleteRequest {
    uint32 volume_id = 1;
    string collection = 2;
    uint64 file_key = 3;
    uint32 version = 4;
}
message VolumeEcBlobDeleteResponse {
}

message VolumeEcShardsToVolumeRequest {
    uint32 volume_id = 1;
    string collection = 2;
    // Staged mode: the caller already decoded the EC shards off-box and pushed
    // .dat/.idx/.vif as <base><ext>.copying to this server (ReceiveFile
    // staged-new-volume mode). Adopt them as a normal volume instead of decoding
    // local EC shards in place, so <vid> is never registered as both EC and
    // normal on one disk.
    bool from_staged = 3;
    string disk_type = 4; // target medium's disk location for the normal volume (staged mode)
}
message VolumeEcShardsToVolumeResponse {
}

message VolumeEcShardsInfoRequest {
    uint32 volume_id = 1;
}
message VolumeEcShardsInfoResponse {
    repeated EcShardInfo ec_shard_infos = 1;
    uint64 volume_size = 2;
    uint64 file_count = 3;
    uint64 file_deleted_count = 4;
    EcShardConfig ec_shard_config = 10;  // the layout this holder serves reads through; a binary predating a field reports it as unset
}

message EcShardInfo {
    uint32 shard_id = 1;
    int64 size = 2;
    string collection = 3;
    uint32 volume_id = 4;
}

message ReadVolumeFileStatusRequest {
    uint32 volume_id = 1;
}
message ReadVolumeFileStatusResponse {
    uint32 volume_id = 1;
    uint64 idx_file_timestamp_seconds = 2;
    uint64 idx_file_size = 3;
    uint64 dat_file_timestamp_seconds = 4;
    uint64 dat_file_size = 5;
    uint64 file_count = 6;
    uint32 compaction_revision = 7;
    string collection = 8;
    string disk_type = 9;
    VolumeInfo volume_info = 10;
    uint32 version = 11;
    uint64 expire_at_sec = 12; // TTL expiry counted from the last write; 0 when the volume never expires
}

message DiskStatus {
    string dir = 1;
    uint64 all = 2;
    uint64 used = 3;
    uint64 free = 4;
    float percent_free = 5;
    float percent_used = 6;
    string disk_type = 7;
    string error = 8;
}

message MemStatus {
    int32 goroutines = 1;
    uint64 all = 2;
    uint64 used = 3;
    uint64 free = 4;
    uint64 self = 5;
    uint64 heap = 6;
    uint64 stack = 7;
}

// tired storage on volume servers
message RemoteFile {
    string backend_type = 1;
    string backend_id = 2;
    string key = 3;
    uint64 offset = 4;
    uint64 file_size = 5;
    uint64 modified_time = 6;
    string extension = 7;
}
message VolumeInfo {
    repeated RemoteFile files = 1;
    uint32 version = 2;
    string replication = 3;
    uint32 bytes_offset = 4;
    int64 dat_file_size = 5; // store the original dat file size
    uint64 expire_at_sec = 6; // expiration time of ec volume
    bool read_only = 7;
    EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4)
    bool read_only_can_delete = 9; // with read_only: writes are rejected but deletes still land
}

// EcShardConfig specifies erasure coding shard configuration
message EcShardConfig {
    uint32 data_shards = 1;   // Number of data shards (e.g., 10)
    uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
    int64 encode_ts_ns = 3;   // encode time (unix nanos); a read served from a shard of a different encode run is rejected
    int64 block_size = 4;     // uniform block layout: each shard is a single contiguous block of this many bytes; 0 = legacy 1GiB/1MiB two-tier layout
}

// EcBitrotProtection is the entire content of a bitrot checksum sidecar
// (<base>.ecsum for the legacy generation, <base>.ecsum.v<N> for vacuum
// generation N). On disk it is wrapped in a fixed header carrying a CRC32C
// over this serialized payload (see weed/storage/erasure_coding/ec_bitrot.go).
message EcBitrotProtection {
    ChecksumAlgorithm algorithm = 1;       // CRC32C (Castagnoli)
    uint32 block_size = 2;                  // bytes per checksum block; default 16777216 (16 MiB), a power-of-two multiple of 1 MiB
    uint32 generation = 3;                  // EC vacuum generation these checksums describe (0 = legacy/fresh); must match the sidecar filename version
    EcShardConfig ec_shard_config = 4;      // data/parity shard counts at encode time
    repeated EcShardChecksums shards = 5;   // one entry per shard id in the active layout
    bytes encode_uuid = 6;                  // random per-encode identity, for stale-sidecar detection across in-place re-encodes
}

message EcShardChecksums {
    uint32 shard_id = 1;     // 0..MaxShardCount-1 (custom EC ratios go up to 32)
    int64 covered_size = 2;  // shard byte length these checksums cover (must equal the on-disk shard length)
    bytes block_crc32c = 3;  // packed little-endian uint32[] = ceil(covered_size/block_size) entries
}

enum ChecksumAlgorithm {
    CHECKSUM_NONE = 0;
    CHECKSUM_CRC32C = 1;
}
message OldVersionVolumeInfo {
    repeated RemoteFile files = 1;
    uint32 version = 2;
    string replication = 3;
    uint32 BytesOffset = 4;
    int64 dat_file_size = 5; // store the original dat file size
    uint64 DestroyTime = 6; // expiration time of ec volume
    bool read_only = 7;
}

// tiered storage
message VolumeTierMoveDatToRemoteRequest {
    uint32 volume_id = 1;
    string collection = 2;
    string destination_backend_name = 3;
    bool keep_local_dat_file = 4;
    int32 concurrency = 5; // multipart upload concurrency, 0 = backend default (5 for S3)
}
message VolumeTierMoveDatToRemoteResponse {
    int64 processed = 1;
    float processedPercentage = 2;
}

message VolumeTierMoveDatFromRemoteRequest {
    uint32 volume_id = 1;
    string collection = 2;
    bool keep_remote_dat_file = 3;
    int32 concurrency = 4; // multipart download concurrency, 0 = backend default (5 for S3)
}
message VolumeTierMoveDatFromRemoteResponse {
    int64 processed = 1;
    float processedPercentage = 2;
}

message VolumeServerStatusRequest {

}
message VolumeServerStatusResponse {
    repeated DiskStatus disk_statuses = 1;
    MemStatus memory_status = 2;
    string version = 3;
    string data_center = 4;
    string rack = 5;
    VolumeServerState state = 6;
}

message VolumeServerLeaveRequest {
}
message VolumeServerLeaveResponse {
}

// remote storage
message FetchAndWriteNeedleRequest {
    uint32 volume_id = 1;
    uint64 needle_id = 2;
    uint32 cookie = 3;
    int64 offset = 4;
    int64 size = 5;
    message Replica {
        string url = 1;
        string public_url = 2;
        int32 grpc_port = 3;
    }
    repeated Replica replicas = 6;
    string auth = 7;
    int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
    // remote conf
    remote_pb.RemoteConf remote_conf = 15;
    remote_pb.RemoteStorageLocation remote_location = 16;
}
message FetchAndWriteNeedleResponse {
    string e_tag = 1;
}

enum VolumeScrubMode {
    UNKNOWN = 0;
    INDEX = 1;
    FULL = 2;
    LOCAL = 3;
    CHECKSUM = 4; // EC only: verify each local shard's raw bytes against the bitrot checksum sidecar
    READS = 5;    // like FULL, but EC intervals no shard can serve are reconstructed from parity
}

message ScrubVolumeRequest {
    VolumeScrubMode mode = 1;
    // optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
    repeated uint32 volume_ids = 2;
    bool mark_broken_volumes_readonly = 3;
}
message ScrubVolumeResponse {
    uint64 total_volumes = 1;
    uint64 total_files = 2;
    repeated uint32 broken_volume_ids = 3;
    repeated string details = 4;
}

message ScrubEcVolumeRequest {
    VolumeScrubMode mode = 1;
    // optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
    repeated uint32 volume_ids = 2;
    bool force_deleted_needles_check = 3; // FULL mode only; may report false positives when EC indexes disagree
}
message ScrubEcVolumeResponse {
    uint64 total_volumes = 1;
    uint64 total_files = 2;
    repeated uint32 broken_volume_ids = 3;
    repeated EcShardInfo broken_shard_infos = 4;
    repeated string details = 5;
}

// select on volume servers
message QueryRequest {
    repeated string selections = 1;
    repeated string from_file_ids = 2;
    message Filter {
        string field = 1;
        string operand = 2;
        string value = 3;
    }
    Filter filter = 3;

    message InputSerialization {
        // NONE | GZIP | BZIP2
        string compression_type = 1;
        message CSVInput {
            string file_header_info = 1; // Valid values: NONE | USE | IGNORE
            string record_delimiter = 2; // Default: \n
            string field_delimiter = 3; // Default: ,
            string quote_character = 4; // Default: "
            string quote_escape_character = 5; // Default: "
            string comments = 6; // Default: #
            // If true, records might contain record delimiters within quote characters
            bool allow_quoted_record_delimiter = 7; // default False.
        }
        message JSONInput {
            string type = 1; // Valid values: DOCUMENT | LINES
        }
        message ParquetInput {
        }

        CSVInput csv_input = 2;
        JSONInput json_input = 3;
        ParquetInput parquet_input = 4;
    }
    InputSerialization input_serialization = 4;

    message OutputSerialization {
        message CSVOutput {
            string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
            string record_delimiter = 2; // Default: \n
            string field_delimiter = 3; // Default: ,
            string quote_character = 4; // Default: "
            string quote_escape_character = 5; // Default: "
        }
        message JSONOutput {
            string record_delimiter = 1;
        }

        CSVOutput csv_output = 2;
        JSONOutput json_output = 3;
    }

    OutputSerialization output_serialization = 5;
}
message QueriedStripe {
    bytes records = 1;
}

message VolumeNeedleStatusRequest {
    uint32 volume_id = 1;
    uint64 needle_id = 2;
}
message VolumeNeedleStatusResponse {
    uint64 needle_id = 1;
    uint32 cookie = 2;
    uint32 size = 3;
    uint64 last_modified = 4;
    uint32 crc = 5;
    string ttl = 6;
}

message PingRequest {
    string target = 1; // default to ping itself
    string target_type = 2;
}
message PingResponse {
    int64 start_time_ns = 1;
    int64 remote_time_ns = 2;
    int64 stop_time_ns = 3;
}