{
  "uid": "forze-operations",
  "title": "Forze / Operations",
  "description": "Operation throughput, outcome split and latency, plus durable runs and job progress.",
  "tags": [
    "forze"
  ],
  "schemaVersion": 39,
  "version": 1,
  "editable": true,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timezone": "",
  "templating": {
    "list": [
      {
        "name": "DS_PROMETHEUS",
        "type": "datasource",
        "query": "prometheus",
        "label": "Datasource",
        "current": {
          "text": "Prometheus",
          "value": "prometheus"
        },
        "hide": 0,
        "refresh": 1
      },
      {
        "name": "service",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": "label_values(job)",
        "label": "Service",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "includeAll": true,
        "multi": true,
        "refresh": 2,
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Operations",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Operation rate",
      "description": "Every operation the registry executed, whatever the outcome.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 4,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_operations{job=~\"$service\"}[5m]))",
          "legendFormat": "ops/s",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Error ratio",
      "description": "Only forze_outcome=\"error\" \u2014 genuine faults. A 4xx-class domain failure is outcome=\"failed\" and deliberately does not count here.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 4,
        "y": 1,
        "w": 4,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_operations{job=~\"$service\",forze_outcome=\"error\"}[5m])) / clamp_min(sum(rate(forze_operations{job=~\"$service\"}[5m])), 1e-9)",
          "legendFormat": "error ratio",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "By outcome",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 8,
        "y": 1,
        "w": 8,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_outcome) (rate(forze_operations{job=~\"$service\"}[5m]))",
          "legendFormat": "{{forze_outcome}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "By kind",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 16,
        "y": 1,
        "w": 8,
        "h": 5
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_operation_kind) (rate(forze_operations{job=~\"$service\"}[5m]))",
          "legendFormat": "{{forze_operation_kind}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Busiest operations",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 6,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "topk(10, sum by (forze_operation) (rate(forze_operations{job=~\"$service\"}[5m])))",
          "legendFormat": "{{forze_operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Failing operations",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 6,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "topk(10, sum by (forze_operation) (rate(forze_operations{job=~\"$service\",forze_outcome=\"error\"}[5m])))",
          "legendFormat": "{{forze_operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 8,
      "type": "heatmap",
      "title": "Duration distribution",
      "description": "Buckets come from the millisecond ladder bootstrap_telemetry installs.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 14,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (le) (rate(forze_operation_duration_bucket{job=~\"$service\"}[5m]))",
          "legendFormat": "{{le}}",
          "refId": "A",
          "format": "heatmap"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms",
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "calculate": false,
        "cellGap": 1,
        "yAxis": {
          "unit": "ms"
        }
      }
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "p95 by operation",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 14,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum by (le, forze_operation) (rate(forze_operation_duration_bucket{job=~\"$service\"}[5m])))",
          "legendFormat": "{{forze_operation}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 10,
      "type": "row",
      "title": "Durable execution",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 22,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "Runs by outcome",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 23,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_durable_outcome) (rate(forze_durable_runs{job=~\"$service\"}[5m]))",
          "legendFormat": "{{forze_durable_outcome}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 12,
      "type": "timeseries",
      "title": "Run duration p95",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 8,
        "y": 23,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum by (le, forze_durable_name) (rate(forze_durable_run_duration_bucket{job=~\"$service\"}[5m])))",
          "legendFormat": "{{forze_durable_name}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ms",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 13,
      "type": "timeseries",
      "title": "Recovered runs and schedule fires",
      "description": "A sustained recovered rate means runs keep being reclaimed from processes that died holding them.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 16,
        "y": 23,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_durable_recovered{job=~\"$service\"}[5m]))",
          "legendFormat": "recovered",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(rate(forze_durable_schedule_fires{job=~\"$service\"}[5m]))",
          "legendFormat": "schedule fires",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 14,
      "type": "row",
      "title": "Job progress",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 30,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 15,
      "type": "timeseries",
      "title": "Stalled jobs",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 31,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by (forze_job_kind) (forze_jobs_stalled{job=~\"$service\"})",
          "legendFormat": "{{forze_job_kind}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 16,
      "type": "timeseries",
      "title": "Oldest silence",
      "description": "",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 8,
        "y": 31,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max by (forze_job_kind) (forze_jobs_stalled_oldest_silence{job=~\"$service\"})",
          "legendFormat": "{{forze_job_kind}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "mappings": []
        },
        "overrides": []
      }
    },
    {
      "id": 17,
      "type": "stat",
      "title": "Staleness sweep age",
      "description": "Watch this, not only the stalled count: the count is a cache, so a dead sweep freezes it at zero while this climbs. -1 means the sweep has never run.",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 16,
        "y": 31,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "max(forze_jobs_staleness_scan_age{job=~\"$service\"})",
          "legendFormat": "scan age",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "mappings": [
            {
              "type": "value",
              "options": {
                "-1": {
                  "text": "never",
                  "color": "red",
                  "index": 0
                }
              }
            }
          ]
        },
        "overrides": []
      }
    }
  ]
}