{
  "uid": "forze-resilience",
  "title": "Forze / Resilience",
  "description": "Breaker states, resilience event rates, bulkhead pressure and adaptive hedging.",
  "tags": [
    "forze"
  ],
  "schemaVersion": 39,
  "version": 1,
  "editable": true,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timezone": "",
  "templating": {
    "list": [
      {
        "name": "DS_PROMETHEUS",
        "type": "datasource",
        "query": "prometheus",
        "label": "Datasource",
        "current": {
          "text": "Prometheus",
          "value": "prometheus"
        },
        "hide": 0,
        "refresh": 1
      },
      {
        "name": "service",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": "label_values(job)",
        "label": "Service",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "includeAll": true,
        "multi": true,
        "refresh": 2,
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Circuit breakers",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Breaker state",
      "description": "0 closed, 1 half-open, 2 open. A breaker that never tripped reports nothing \u2014 closed by absence.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max by (forze_policy, forze_route) (forze_resilience_breaker_state{job=~\"$service\"})",
          "legendFormat": "{{forze_policy}} / {{forze_route}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "mappings": [
            {
              "type": "value",
              "options": {
                "0": {
                  "text": "closed",
                  "color": "green",
                  "index": 0
                },
                "1": {
                  "text": "half-open",
                  "color": "orange",
                  "index": 1
                },
                "2": {
                  "text": "open",
                  "color": "red",
                  "index": 2
                }
              }
            }
          ]
        },
        "overrides": []
      }
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Events by kind",
      "description": "breaker_open counts the open transition and every admission rejected while open, so its rate tracks shed load.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_event) (rate(forze_resilience_events{job=~\"$service\"}[5m]))",
          "legendFormat": "{{forze_event}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 4,
      "type": "row",
      "title": "Bulkheads",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 9,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Queue depth vs adaptive limit",
      "description": "Queue depth pinned at the limit is the shape that precedes shedding.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 10,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_policy, forze_route) (forze_resilience_bulkhead_queue_depth{job=~\"$service\"})",
          "legendFormat": "queued {{forze_policy}} / {{forze_route}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_policy, forze_route) (forze_resilience_bulkhead_limit{job=~\"$service\"})",
          "legendFormat": "limit {{forze_policy}} / {{forze_route}}",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Rejections and backoff",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 10,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_event) (rate(forze_resilience_events{job=~\"$service\",forze_event=~\"bulkhead_.*|rate_limited|budget_exhausted\"}[5m]))",
          "legendFormat": "{{forze_event}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 7,
      "type": "row",
      "title": "Hedging",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 18,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Adaptive hedge delay",
      "description": "The windowed P\u00b2 quantile estimate, clamped by the strategy's floor and cap.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 19,
        "w": 24,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max by (forze_policy, forze_route) (forze_resilience_hedge_delay{job=~\"$service\"})",
          "legendFormat": "{{forze_policy}} / {{forze_route}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "mappings": []
        },
        "overrides": []
      }
    }
  ]
}