{
  "__inputs": [],
  "title": "Warp homeserver",
  "uid": "warp-homeserver",
  "description": "Warp: HTTP, rooms, sync, database and runtime. Built on the metrics /metrics exports.",
  "tags": [
    "warp",
    "matrix"
  ],
  "timezone": "browser",
  "editable": true,
  "schemaVersion": 39,
  "version": 1,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "label": "Data source",
        "current": {},
        "hide": 0
      },
      {
        "name": "job",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "label": "Job",
        "query": {
          "query": "label_values(warp_build_info, job)",
          "refId": "job"
        },
        "definition": "label_values(warp_build_info, job)",
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "refresh": 2,
        "current": {}
      },
      {
        "name": "instance",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "label": "Instance",
        "query": {
          "query": "label_values(warp_build_info{job=~\"$job\"}, instance)",
          "refId": "instance"
        },
        "definition": "label_values(warp_build_info{job=~\"$job\"}, instance)",
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "refresh": 2,
        "current": {}
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Overview",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Version",
      "description": "The Warp version each instance reports.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max by (version) (warp_build_info{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "{{version}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area",
        "textMode": "name"
      }
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Requests / s",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 4,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 4,
      "type": "stat",
      "title": "5xx ratio",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 8,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\", status=~\"5..\"}[$__rate_interval])) / sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 5,
      "type": "stat",
      "title": "Events / min",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(rate(warp_events_persisted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 60",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 6,
      "type": "stat",
      "title": "Waiting syncs",
      "description": "Clients blocked in a /sync or sliding sync long poll: roughly the connected clients.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 16,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(warp_sync_waiting{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 7,
      "type": "stat",
      "title": "Federation",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 20,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "max(warp_federation_enabled{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "type": "value",
              "options": {
                "0": {
                  "text": "off",
                  "color": "text"
                },
                "1": {
                  "text": "on",
                  "color": "green"
                }
              }
            }
          ]
        },
        "overrides": []
      },
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "colorMode": "value",
        "graphMode": "area"
      }
    },
    {
      "id": 8,
      "type": "row",
      "title": "HTTP",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 5,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "Requests by route",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 6,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk(10, sum by (route) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
          "legendFormat": "{{route}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 10,
      "type": "timeseries",
      "title": "Responses by status",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 6,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (status) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
          "legendFormat": "{{status}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 40,
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "p95 latency by route (long polls excluded)",
      "description": "Sync endpoints block for up to their timeout by design, so they are left out.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 15,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "topk(10, histogram_quantile(0.95, sum by (le, route) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval]))))",
          "legendFormat": "{{route}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 12,
      "type": "timeseries",
      "title": "Latency quantiles, all routes but sync",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 15,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.5, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))",
          "legendFormat": "p95",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "histogram_quantile(0.99, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))",
          "legendFormat": "p99",
          "refId": "C"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 13,
      "type": "timeseries",
      "title": "In-flight requests",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 24,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (instance) (warp_http_requests_in_flight{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 14,
      "type": "timeseries",
      "title": "Federation requests",
      "description": "Requests from other servers. Empty while federation is off.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 24,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (route) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\", route=~\".*/_matrix/(federation|key)/.*\"}[$__rate_interval]))",
          "legendFormat": "{{route}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 15,
      "type": "row",
      "title": "Rooms and sync",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 32,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 16,
      "type": "timeseries",
      "title": "Events persisted",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 33,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (origin, kind) (rate(warp_events_persisted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
          "legendFormat": "{{origin}} {{kind}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 40,
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 17,
      "type": "timeseries",
      "title": "Sync connections",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 33,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(warp_sync_waiting{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "waiting",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum(warp_sync_sliding_connections{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "sliding sync connections",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 18,
      "type": "row",
      "title": "Database",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 41,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 19,
      "type": "timeseries",
      "title": "Connections",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 42,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (db_name) (go_sql_open_connections{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "open {{db_name}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (db_name) (go_sql_in_use_connections{job=~\"$job\", instance=~\"$instance\"})",
          "legendFormat": "in use {{db_name}}",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 20,
      "type": "timeseries",
      "title": "Waiting for a connection",
      "description": "Seconds per second spent waiting for a pooled connection. SQLite and PocketBase have one writer, so some waiting under load is expected.",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 12,
        "y": 42,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "sum by (db_name) (rate(go_sql_wait_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
          "legendFormat": "{{db_name}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 21,
      "type": "row",
      "title": "Go runtime",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 50,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 22,
      "type": "timeseries",
      "title": "Goroutines",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 0,
        "y": 51,
        "w": 8,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "go_goroutines{job=~\"$job\", instance=~\"$instance\"}",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 23,
      "type": "timeseries",
      "title": "Heap in use",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 8,
        "y": 51,
        "w": 8,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 24,
      "type": "timeseries",
      "title": "CPU",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "gridPos": {
        "x": 16,
        "y": 51,
        "w": 8,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
          "legendFormat": "{{instance}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "custom": {
            "lineWidth": 1,
            "fillOpacity": 10,
            "stacking": {
              "mode": "none"
            }
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "mean",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    }
  ]
}