{
  "uid": "forze-realtime",
  "title": "Forze / Realtime",
  "description": "Gateway delivery outcomes, backplane freshness and the offline mailbox.",
  "tags": [
    "forze"
  ],
  "schemaVersion": 39,
  "version": 1,
  "editable": true,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timezone": "",
  "templating": {
    "list": [
      {
        "name": "DS_PROMETHEUS",
        "type": "datasource",
        "query": "prometheus",
        "label": "Datasource",
        "current": {
          "text": "Prometheus",
          "value": "prometheus"
        },
        "hide": 0,
        "refresh": 1
      },
      {
        "name": "service",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": "label_values(job)",
        "label": "Service",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "includeAll": true,
        "multi": true,
        "refresh": 2,
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Gateway delivery",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Emitted vs failed",
      "description": "failed climbing while emitted is flat means Socket.IO \u2014 or its Redis backplane \u2014 stopped taking frames.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_realtime_channel) (rate(forze_realtime_gateway_emitted{job=~\"$service\"}[5m]))",
          "legendFormat": "emitted {{forze_realtime_channel}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_realtime_channel) (rate(forze_realtime_gateway_emit_failed{job=~\"$service\"}[5m]))",
          "legendFormat": "failed {{forze_realtime_channel}}",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Skips and rejections",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_presence_skipped{job=~\"$service\"}[5m]))",
          "legendFormat": "presence skipped",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_dedup_skipped{job=~\"$service\"}[5m]))",
          "legendFormat": "dedup skipped",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_admission_rejected{job=~\"$service\"}[5m]))",
          "legendFormat": "admission rejected",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_mailboxed{job=~\"$service\"}[5m]))",
          "legendFormat": "mailboxed",
          "refId": "D"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "Dropped deliveries",
      "description": "Every poisoned increment is a durable delivery nobody will ever receive.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 9,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_poisoned{job=~\"$service\"}[5m]))",
          "legendFormat": "poisoned",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_untenanted_dropped{job=~\"$service\"}[5m]))",
          "legendFormat": "untenanted dropped",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_gateway_bridge_failed{job=~\"$service\"}[5m]))",
          "legendFormat": "bridge failed",
          "refId": "C"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Backplane freshness",
      "description": "-1 means no probe has ever succeeded: wiring, not outage.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 9,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max by (forze_realtime_channel) (forze_realtime_backplane_seconds_since_ok{job=~\"$service\"})",
          "legendFormat": "since ok {{forze_realtime_channel}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max by (forze_realtime_channel) (forze_realtime_backplane_consecutive_failures{job=~\"$service\"})",
          "legendFormat": "failures {{forze_realtime_channel}}",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 6,
      "type": "row",
      "title": "Offline mailbox",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 16,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Store and forward",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 17,
        "w": 16,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_mailbox_stored{job=~\"$service\"}[5m]))",
          "legendFormat": "stored",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_mailbox_replayed{job=~\"$service\"}[5m]))",
          "legendFormat": "replayed",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_mailbox_trimmed{job=~\"$service\"}[5m]))",
          "legendFormat": "trimmed",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_realtime_mailbox_acked{job=~\"$service\"}[5m]))",
          "legendFormat": "acked",
          "refId": "D"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 8,
      "type": "stat",
      "title": "Overflowed replays",
      "description": "Each one is a device that fell more than cap entries behind and lost signals for good.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 16,
        "y": 17,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(increase(forze_realtime_mailbox_overflowed{job=~\"$service\"}[1h]))",
          "legendFormat": "overflowed (1h)",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "mappings": []
        },
        "overrides": []
      }
    }
  ]
}