seaweedfs/weed/pb/plugin.proto

syntax = "proto3";

package plugin;

option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb";

import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

// PluginControlService is the admin-facing stream API for external workers.
// Workers initiate and keep this stream alive; all control plane traffic flows through it.
service PluginControlService {
  rpc WorkerStream(stream WorkerToAdminMessage) returns (stream AdminToWorkerMessage);
}

// WorkerToAdminMessage carries worker-originated events and responses.
message WorkerToAdminMessage {
  string worker_id = 1;
  google.protobuf.Timestamp sent_at = 2;

  oneof body {
    WorkerHello hello = 10;
    WorkerHeartbeat heartbeat = 11;
    WorkerAcknowledge acknowledge = 12;
    ConfigSchemaResponse config_schema_response = 13;
    DetectionProposals detection_proposals = 14;
    DetectionComplete detection_complete = 15;
    JobProgressUpdate job_progress_update = 16;
    JobCompleted job_completed = 17;
    WorkerObservations observations = 18;
    ObjectPreviewResponse object_preview_response = 19;
  }
}

// AdminToWorkerMessage carries commands and lifecycle notifications from admin.
message AdminToWorkerMessage {
  string request_id = 1;
  google.protobuf.Timestamp sent_at = 2;

  oneof body {
    AdminHello hello = 10;
    RequestConfigSchema request_config_schema = 11;
    RunDetectionRequest run_detection_request = 12;
    ExecuteJobRequest execute_job_request = 13;
    CancelRequest cancel_request = 14;
    AdminShutdown shutdown = 15;
    RequestObjectPreview request_object_preview = 16;
  }
}

message WorkerHello {
  string worker_id = 1;
  string worker_instance_id = 2;
  string address = 3;
  string worker_version = 4;
  string protocol_version = 5;
  repeated JobTypeCapability capabilities = 6;
  map<string, string> metadata = 7;
}

message AdminHello {
  bool accepted = 1;
  string message = 2;
  int32 heartbeat_interval_seconds = 3;
  int32 reconnect_delay_seconds = 4;
}

message WorkerHeartbeat {
  string worker_id = 1;
  repeated RunningWork running_work = 2;
  int32 detection_slots_used = 3;
  int32 detection_slots_total = 4;
  int32 execution_slots_used = 5;
  int32 execution_slots_total = 6;
  map<string, int32> queued_jobs_by_type = 7;
  map<string, string> metadata = 8;
}

message WorkerAcknowledge {
  string request_id = 1;
  bool accepted = 2;
  string message = 3;
}

message RunningWork {
  string work_id = 1;
  WorkKind kind = 2;
  string job_type = 3;
  JobState state = 4;
  double progress_percent = 5;
  string stage = 6;
}

message JobTypeCapability {
  string job_type = 1;
  bool can_detect = 2;
  bool can_execute = 3;
  int32 max_detection_concurrency = 4;
  int32 max_execution_concurrency = 5;
  string display_name = 6;
  string description = 7;
  int32 weight = 8;
}

message RequestConfigSchema {
  string job_type = 1;
  bool force_refresh = 2;
}

message ConfigSchemaResponse {
  string request_id = 1;
  string job_type = 2;
  bool success = 3;
  string error_message = 4;
  JobTypeDescriptor job_type_descriptor = 5;
}

// JobTypeDescriptor defines one job type contract, including UI schema and defaults.
message JobTypeDescriptor {
  string job_type = 1;
  string display_name = 2;
  string description = 3;
  string icon = 4;
  uint32 descriptor_version = 5;

  // Admin-owned options such as detection frequency and dispatch concurrency.
  ConfigForm admin_config_form = 6;
  // Worker-owned options used during detection and execution.
  ConfigForm worker_config_form = 7;

  AdminRuntimeDefaults admin_runtime_defaults = 8;
  map<string, ConfigValue> worker_default_values = 9;
  // Worker config field containing a cooldown in seconds after the latest successful job.
  string detection_cooldown_config_field = 10;
}

message ConfigForm {
  string form_id = 1;
  string title = 2;
  string description = 3;
  repeated ConfigSection sections = 4;
  map<string, ConfigValue> default_values = 5;
}

message ConfigSection {
  string section_id = 1;
  string title = 2;
  string description = 3;
  repeated ConfigField fields = 4;
}

message ConfigField {
  string name = 1;
  string label = 2;
  string description = 3;
  string help_text = 4;
  string placeholder = 5;

  ConfigFieldType field_type = 6;
  ConfigWidget widget = 7;

  bool required = 8;
  bool read_only = 9;
  bool sensitive = 10;

  ConfigValue min_value = 11;
  ConfigValue max_value = 12;

  repeated ConfigOption options = 13;
  repeated ValidationRule validation_rules = 14;

  // Simple visibility dependency: show this field when the referenced field equals value.
  string visible_when_field = 15;
  ConfigValue visible_when_equals = 16;
}

message ConfigOption {
  string value = 1;
  string label = 2;
  string description = 3;
  bool disabled = 4;
}

message ValidationRule {
  ValidationRuleType type = 1;
  string expression = 2;
  string error_message = 3;
}

message ConfigValue {
  oneof kind {
    bool bool_value = 1;
    int64 int64_value = 2;
    double double_value = 3;
    string string_value = 4;
    bytes bytes_value = 5;
    google.protobuf.Duration duration_value = 6;
    StringList string_list = 7;
    Int64List int64_list = 8;
    DoubleList double_list = 9;
    BoolList bool_list = 10;
    ValueList list_value = 11;
    ValueMap map_value = 12;
  }
}

message StringList {
  repeated string values = 1;
}

message Int64List {
  repeated int64 values = 1;
}

message DoubleList {
  repeated double values = 1;
}

message BoolList {
  repeated bool values = 1;
}

message ValueList {
  repeated ConfigValue values = 1;
}

message ValueMap {
  map<string, ConfigValue> fields = 1;
}

message AdminRuntimeDefaults {
  bool enabled = 1;
  int32 detection_interval_minutes = 2;
  int32 detection_timeout_seconds = 3;
  int32 max_jobs_per_detection = 4;
  int32 global_execution_concurrency = 5;
  int32 per_worker_execution_concurrency = 6;
  int32 retry_limit = 7;
  int32 retry_backoff_seconds = 8;
  int32 job_type_max_runtime_seconds = 9;
  // Per-attempt execution deadline for one job. Distinct from
  // detection_timeout (which bounds the detection scan) and
  // job_type_max_runtime (a lane-wide budget across all attempts).
  int32 execution_timeout_seconds = 10;
}

message AdminRuntimeConfig {
  bool enabled = 1;
  int32 detection_interval_minutes = 2;
  int32 detection_timeout_seconds = 3;
  int32 max_jobs_per_detection = 4;
  int32 global_execution_concurrency = 5;
  int32 per_worker_execution_concurrency = 6;
  int32 retry_limit = 7;
  int32 retry_backoff_seconds = 8;
  int32 job_type_max_runtime_seconds = 9;
  int32 execution_timeout_seconds = 10;
}

message RunDetectionRequest {
  string request_id = 1;
  string job_type = 2;
  int64 detection_sequence = 3;

  AdminRuntimeConfig admin_runtime = 4;
  map<string, ConfigValue> admin_config_values = 5;
  map<string, ConfigValue> worker_config_values = 6;

  ClusterContext cluster_context = 7;
  google.protobuf.Timestamp last_successful_run = 8;
  int32 max_results = 9;
}

message DetectionProposals {
  string request_id = 1;
  string job_type = 2;
  repeated JobProposal proposals = 3;
  bool has_more = 4;
}

message DetectionComplete {
  string request_id = 1;
  string job_type = 2;
  bool success = 3;
  string error_message = 4;
  int32 total_proposals = 5;
  string skipped_reason = 6;
  google.protobuf.Timestamp eligible_after = 7;
}

message JobProposal {
  string proposal_id = 1;
  string dedupe_key = 2;
  string job_type = 3;
  JobPriority priority = 4;
  string summary = 5;
  string detail = 6;
  map<string, ConfigValue> parameters = 7;
  map<string, string> labels = 8;
  google.protobuf.Timestamp not_before = 9;
  google.protobuf.Timestamp expires_at = 10;
}

message ExecuteJobRequest {
  string request_id = 1;
  JobSpec job = 2;
  AdminRuntimeConfig admin_runtime = 3;
  map<string, ConfigValue> admin_config_values = 4;
  map<string, ConfigValue> worker_config_values = 5;
  ClusterContext cluster_context = 6;
  int32 attempt = 7;
}

message JobSpec {
  string job_id = 1;
  string job_type = 2;
  string dedupe_key = 3;
  JobPriority priority = 4;
  string summary = 5;
  string detail = 6;
  map<string, ConfigValue> parameters = 7;
  map<string, string> labels = 8;
  google.protobuf.Timestamp created_at = 9;
  google.protobuf.Timestamp scheduled_at = 10;
}

message JobProgressUpdate {
  string request_id = 1;
  string job_id = 2;
  string job_type = 3;
  JobState state = 4;
  double progress_percent = 5;
  string stage = 6;
  string message = 7;
  map<string, ConfigValue> metrics = 8;
  repeated ActivityEvent activities = 9;
  google.protobuf.Timestamp updated_at = 10;
}

message JobCompleted {
  string request_id = 1;
  string job_id = 2;
  string job_type = 3;
  bool success = 4;
  string error_message = 5;
  JobResult result = 6;
  repeated ActivityEvent activities = 7;
  google.protobuf.Timestamp completed_at = 8;
}

message JobResult {
  map<string, ConfigValue> output_values = 1;
  string summary = 2;
}

message ClusterContext {
  repeated string master_grpc_addresses = 1;
  // Filers in pb.ServerAddress form (host:httpPort.grpcPort). Consumers
  // convert to a gRPC or HTTP address as needed; see weed/pb/server_address.go.
  repeated string filer_addresses = 2;
  repeated string volume_grpc_addresses = 3;
  map<string, string> metadata = 4;
  repeated string s3_grpc_addresses = 5;
}

message ActivityEvent {
  ActivitySource source = 1;
  string message = 2;
  string stage = 3;
  map<string, ConfigValue> details = 4;
  google.protobuf.Timestamp created_at = 5;
}

message CancelRequest {
  string target_id = 1;
  WorkKind target_kind = 2;
  string reason = 3;
  bool force = 4;
}

message AdminShutdown {
  string reason = 1;
  int32 grace_period_seconds = 2;
}

// PersistedJobTypeConfig is the admin-side on-disk model per job type.
message PersistedJobTypeConfig {
  string job_type = 1;
  uint32 descriptor_version = 2;
  map<string, ConfigValue> admin_config_values = 3;
  map<string, ConfigValue> worker_config_values = 4;
  AdminRuntimeConfig admin_runtime = 5;
  google.protobuf.Timestamp updated_at = 6;
  string updated_by = 7;
}

enum WorkKind {
  WORK_KIND_UNSPECIFIED = 0;
  WORK_KIND_DETECTION = 1;
  WORK_KIND_EXECUTION = 2;
}

enum JobPriority {
  JOB_PRIORITY_UNSPECIFIED = 0;
  JOB_PRIORITY_LOW = 1;
  JOB_PRIORITY_NORMAL = 2;
  JOB_PRIORITY_HIGH = 3;
  JOB_PRIORITY_CRITICAL = 4;
}

enum JobState {
  JOB_STATE_UNSPECIFIED = 0;
  JOB_STATE_PENDING = 1;
  JOB_STATE_ASSIGNED = 2;
  JOB_STATE_RUNNING = 3;
  JOB_STATE_SUCCEEDED = 4;
  JOB_STATE_FAILED = 5;
  JOB_STATE_CANCELED = 6;
}

enum ConfigFieldType {
  CONFIG_FIELD_TYPE_UNSPECIFIED = 0;
  CONFIG_FIELD_TYPE_BOOL = 1;
  CONFIG_FIELD_TYPE_INT64 = 2;
  CONFIG_FIELD_TYPE_DOUBLE = 3;
  CONFIG_FIELD_TYPE_STRING = 4;
  CONFIG_FIELD_TYPE_BYTES = 5;
  CONFIG_FIELD_TYPE_DURATION = 6;
  CONFIG_FIELD_TYPE_ENUM = 7;
  CONFIG_FIELD_TYPE_LIST = 8;
  CONFIG_FIELD_TYPE_OBJECT = 9;
}

enum ConfigWidget {
  CONFIG_WIDGET_UNSPECIFIED = 0;
  CONFIG_WIDGET_TOGGLE = 1;
  CONFIG_WIDGET_TEXT = 2;
  CONFIG_WIDGET_TEXTAREA = 3;
  CONFIG_WIDGET_NUMBER = 4;
  CONFIG_WIDGET_SELECT = 5;
  CONFIG_WIDGET_MULTI_SELECT = 6;
  CONFIG_WIDGET_DURATION = 7;
  CONFIG_WIDGET_PASSWORD = 8;
}

enum ValidationRuleType {
  VALIDATION_RULE_TYPE_UNSPECIFIED = 0;
  VALIDATION_RULE_TYPE_REGEX = 1;
  VALIDATION_RULE_TYPE_MIN_LENGTH = 2;
  VALIDATION_RULE_TYPE_MAX_LENGTH = 3;
  VALIDATION_RULE_TYPE_MIN_ITEMS = 4;
  VALIDATION_RULE_TYPE_MAX_ITEMS = 5;
  VALIDATION_RULE_TYPE_CUSTOM = 6;
}

enum ActivitySource {
  ACTIVITY_SOURCE_UNSPECIFIED = 0;
  ACTIVITY_SOURCE_ADMIN = 1;
  ACTIVITY_SOURCE_DETECTOR = 2;
  ACTIVITY_SOURCE_EXECUTOR = 3;
}

// WorkerObservations reports what a worker learned about the objects it
// inspected. A worker opens a table to decide whether it needs a job; what it
// saw is worth keeping either way, because for a format the cluster cannot
// read, the worker is the only thing that can describe it.
//
// Observations are not work. Admin caches them and serves them back, so a
// worker that goes away leaves its last ones standing rather than blanking a
// page.
message WorkerObservations {
  string job_type = 1;
  repeated ObjectObservation observations = 2;
}

// ObjectObservation is one object as a worker last saw it.
message ObjectObservation {
  // Identifier of the object, e.g. ["bucket", "namespace", "table"].
  repeated string object_id = 1;
  // What kind of object this is, e.g. "table".
  string object_kind = 2;
  // The object's format, e.g. "LANCE", so a reader can tell whose observation
  // this is without parsing the attributes.
  string format = 3;
  // Whatever the worker can cheaply say: schema, row count, fragment count.
  // The keys are the worker's to choose.
  map<string, ConfigValue> attributes = 4;
  google.protobuf.Timestamp observed_at = 5;
}

// RequestObjectPreview asks a worker for sample rows of an object admin cannot
// read itself. Unlike an observation this is not cached: it is fetched when
// someone opens the page, because rows are the object's data rather than a
// description of it, and holding a copy in admin is neither fresh nor its
// business.
message RequestObjectPreview {
  repeated string object_id = 1;
  // The format admin believes this object is, so a worker that does not own it
  // can decline instead of guessing.
  string format = 2;
  int32 row_limit = 3;
}

// ObjectPreviewResponse carries the sample back, already rendered as text. The
// worker is the only thing that knows the object's types, so it formats them;
// admin displays what it is given.
message ObjectPreviewResponse {
  string request_id = 6;
  bool success = 1;
  string error_message = 2;
  repeated string columns = 3;
  repeated PreviewRow rows = 4;
  // Rows in the object, which is not the number sampled.
  int64 total_rows = 5;
}

message PreviewRow {
  repeated string values = 1;
}