{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, { "type": "panel", "id": "heatmap", "name": "Heatmap", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "pg_trickle stream table health, refresh performance, CDC buffer metrics, and per-table drill-down", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "green", "text": "OK" }, "1": { "color": "yellow", "text": "WARNING" }, "2": { "color": "red", "text": "CRITICAL" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value_and_name" }, "title": "Cluster Health", "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_health_health_status_code", "legendFormat": "Health", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Total Stream Tables", "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_health_total_stream_tables", "legendFormat": "Tables", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Stale Tables", "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_health_stale_tables", "legendFormat": "Stale", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Tables With Errors", "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_health_error_tables", "legendFormat": "Errors", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 5, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value_and_name" }, "title": "Scheduler", "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_health_scheduler_running", "legendFormat": "Scheduler", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 6, "options": { "displayLabels": [ "percent" ], "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, "pieType": "donut", "tooltip": { "mode": "single", "sort": "none" } }, "title": "Tables by Status", "type": "piechart", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_status_counts_stream_tables_total", "legendFormat": "{{ status }}", "refId": "A" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "title": "Refresh Performance", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 10, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "Avg Refresh Duration by Table", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_avg_duration_ms > 0", "legendFormat": "{{ schema }}.{{ name }}", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "pointSize": 5, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 11, "options": { "legend": { "calcs": [ "sum" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "Failed Refreshes (last hour)", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "increase(pg_trickle_table_stats_failed_refreshes[5m])", "legendFormat": "{{ schema }}.{{ name }}", "refId": "A" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 102, "title": "Staleness Heatmap", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "spanNulls": false }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 20, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "Data Staleness by Table", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_staleness_seconds", "legendFormat": "{{ schema }}.{{ name }}", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "orange", "value": 300 }, { "color": "red", "value": 600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 21, "options": { "calculate": false, "cellGap": 2, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "reverse": false } }, "title": "Staleness Heatmap (all tables)", "type": "heatmap", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_staleness_seconds", "legendFormat": "{{ schema }}.{{ name }}", "refId": "A" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 103, "title": "CDC Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2, "pointSize": 5, "spanNulls": false, "stacking": { "group": "A", "mode": "none" } }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "id": 30, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "CDC Pending Rows by Buffer", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_cdc_buffers_pending_rows", "legendFormat": "{{ stream_table }} \u2190 {{ source_table }}", "refId": "A" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2, "pointSize": 5, "spanNulls": false }, "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 104857600 }, { "color": "red", "value": 1073741824 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "id": 31, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "CDC Buffer Size (bytes)", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_cdc_buffers_buffer_bytes", "legendFormat": "{{ stream_table }} \u2190 {{ source_table }}", "refId": "A" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 104, "title": "Per-Table Drill-Down", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "status" }, "properties": [ { "id": "mappings", "value": [ { "options": { "ACTIVE": { "color": "green", "index": 0 }, "ERROR": { "color": "red", "index": 1 }, "SUSPENDED": { "color": "dark-red", "index": 2 }, "INITIALIZING": { "color": "blue", "index": 3 } }, "type": "value" } ] }, { "id": "custom.displayMode", "value": "color-text" } ] }, { "matcher": { "id": "byName", "options": "stale" }, "properties": [ { "id": "mappings", "value": [ { "options": { "0": { "color": "green", "text": "Fresh", "index": 0 }, "1": { "color": "red", "text": "STALE", "index": 1 } }, "type": "value" } ] }, { "id": "custom.displayMode", "value": "color-background" } ] }, { "matcher": { "id": "byName", "options": "staleness_seconds" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "avg_duration_ms" }, "properties": [ { "id": "unit", "value": "ms" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5000 }, { "color": "red", "value": 30000 } ] } }, { "id": "custom.displayMode", "value": "color-background" } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 33 }, "id": 40, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "consecutive_errors" } ] }, "title": "Stream Table Status", "type": "table", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_consecutive_errors", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_staleness_seconds", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_avg_duration_ms", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "pg_trickle_table_stats_stale", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "D" } ], "transformations": [ { "id": "joinByField", "options": { "byField": "Time", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__ 1": true, "__name__ 2": true, "__name__ 3": true, "__name__ 4": true, "instance 1": true, "instance 2": true, "instance 3": true, "instance 4": true, "job 1": true, "job 2": true, "job 3": true, "job 4": true }, "renameByName": { "schema 1": "schema", "name 1": "name", "status 1": "status", "refresh_mode 1": "refresh_mode", "effective_refresh_mode 1": "effective_refresh_mode", "Value #A": "consecutive_errors", "Value #B": "staleness_seconds", "Value #C": "avg_duration_ms", "Value #D": "stale" } } } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, "spanNulls": false }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "id": 50, "options": { "legend": { "calcs": [ "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "title": "Rows Inserted / Deleted (rate)", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(pg_trickle_table_stats_rows_inserted_total[5m])", "legendFormat": "inserted {{ schema }}.{{ name }}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(pg_trickle_table_stats_rows_deleted_total[5m])", "legendFormat": "deleted {{ schema }}.{{ name }}", "refId": "B" } ] }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "stacking": { "group": "A", "mode": "none" } }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "id": 51, "options": { "legend": { "calcs": [ "sum" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "title": "Successful vs Failed Refreshes (rate)", "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(pg_trickle_table_stats_successful_refreshes[5m])", "legendFormat": "ok {{ schema }}.{{ name }}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "rate(pg_trickle_table_stats_failed_refreshes[5m])", "legendFormat": "fail {{ schema }}.{{ name }}", "refId": "B" } ] } ], "refresh": "30s", "schemaVersion": 38, "tags": [ "pg_trickle", "postgresql", "streaming", "ivm" ], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(pg_trickle_table_stats_staleness_seconds, schema)", "hide": 0, "includeAll": true, "multi": true, "name": "schema", "options": [], "query": { "query": "label_values(pg_trickle_table_stats_staleness_seconds, schema)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query", "label": "Schema" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(pg_trickle_table_stats_staleness_seconds{schema=~\"$schema\"}, name)", "hide": 0, "includeAll": true, "multi": true, "name": "table", "options": [], "query": { "query": "label_values(pg_trickle_table_stats_staleness_seconds{schema=~\"$schema\"}, name)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query", "label": "Table" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "pg_trickle Overview", "uid": "pg-trickle-overview-v1", "version": 1 }