diff --git a/environment/plaintext/docker-compose.yml b/environment/plaintext/docker-compose.yml index 65f11168e..a0d460181 100644 --- a/environment/plaintext/docker-compose.yml +++ b/environment/plaintext/docker-compose.yml @@ -615,7 +615,8 @@ services: profiles: - flink ports: - - "18081:18081" # Expose port 18081 for REST API and Web UI + - "18081:18081" # REST API and Web UI + - "9090:9090" # Prometheus metrics entrypoint: > sh -c "$flink_connectors exec /docker-entrypoint.sh jobmanager" environment: @@ -624,7 +625,8 @@ services: jobmanager.rpc.address: jobmanager rest.port: 18081 taskmanager.numberOfTaskSlots: 4 - + ${GRAFANA_FLINK} + # Flink TaskManager taskmanager: image: flink:${FLINK_TAG} @@ -643,7 +645,7 @@ services: rest.address: jobmanager rest.port: 18081 taskmanager.numberOfTaskSlots: 4 - + ${GRAFANA_FLINK} # Flink SQL Client sql-client: image: flink:${FLINK_TAG} diff --git a/environment/plaintext/grafana/provisioning/dashboards/flink.json b/environment/plaintext/grafana/provisioning/dashboards/flink.json new file mode 100644 index 000000000..52da5127e --- /dev/null +++ b/environment/plaintext/grafana/provisioning/dashboards/flink.json @@ -0,0 +1,7603 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1629750069653, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 113, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_CPU_Load{instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_Status_JVM_CPU_Time{instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Time", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Threads_Count{instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Threads", + "type": "timeseries" + } + ], + "title": "Job Manager (JVM - CPU)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 122, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_Heap_Used{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Heap", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_NonHeap_Used{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Heap", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_Direct_MemoryUsed{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Direct", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 126, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_Mapped_MemoryUsed{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Mapped", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_Metaspace_Used{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Metaspace", + "type": "timeseries" + } + ], + "title": "Job Manager (JVM - Memory Usage)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 118, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_GarbageCollector_PS_MarkSweep_Time", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Mark Sweep Time", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_Status_JVM_GarbageCollector_PS_Scavenge_Time[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Scavenge Time", + "type": "timeseries" + } + ], + "title": "Job Manager (JVM - Garbage Collector)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 10, + "panels": [ + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 2, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_numRegisteredTaskManagers{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Task Managers", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 3, + "y": 4 + }, + "id": 31, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_taskSlotsTotal{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Task Slots", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 10, + "y": 4 + }, + "id": 32, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_taskSlotsAvailable{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Available Task Slots", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 17, + "y": 4 + }, + "id": 33, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_numRunningJobs{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Runnings Jobs", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 34, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_numRestarts{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Job Restarts", + "transformations": [], + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 42, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_fullRestarts{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Job Restarts", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 43, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_restartingTime{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Job Restartine Time", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "datasource": "${Source}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 3600000 + }, + { + "color": "dark-orange", + "value": 21600000 + }, + { + "color": "dark-purple", + "value": 86400000 + }, + { + "color": "dark-red", + "value": 259200000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 21, + "x": 0, + "y": 20 + }, + "id": 37, + "interval": null, + "links": [], + "maxDataPoints": null, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.0", + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_uptime{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_id}} ({{instance}})", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Job Uptime", + "type": "stat" + }, + { + "datasource": null, + "description": "", + "gridPos": { + "h": 10, + "w": 3, + "x": 21, + "y": 20 + }, + "id": 46, + "options": { + "content": "# Uptime Description\n\n| Color | Status|\n|---|---|\n| Green | < 1 Hour |\n| Yellow | >= 1 Hour |\n| Orange | >= 6 Hours |\n| Purple | >= 1 Day |\n| Red | >= 3 days |\n", + "mode": "markdown" + }, + "pluginVersion": "8.1.0", + "targets": [ + { + "refId": "A", + "target": "" + } + ], + "title": "Uptime Derscription", + "type": "text" + } + ], + "title": "Job Manager (Slots & Jobs)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 106, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_job_totalNumberOfCheckpoints{job_name=~\"$job_name\", instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Checkpoints", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_job_numberOfCompletedCheckpoints{job_name=~\"$job_name\", instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Completed Checkpoints", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_job_numberOfInProgressCheckpoints{job_name=~\"$job_name\", instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "In Progress Checkpoints", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_jobmanager_job_numberOfFailedCheckpoints{job_name=~\"$job_name\", instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Failed Checkpoints", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_lastCheckpointSize{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Last Checkpoint Size", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_jobmanager_job_lastCheckpointDuration{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Last Checkpoint Duration", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "xx{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "type": "timeseries" + } + ], + "title": "Job Manager (Checkpoints)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 91, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_CPU_Load{tm_id=~\"${tm_id}\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "increase(flink_taskmanager_Status_JVM_CPU_Time{tm_id=~\"$tm_id\"}[1m])", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Time", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Threads_Count{tm_id=~\"${tm_id}\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Threads", + "type": "timeseries" + } + ], + "title": "Task Manager (JVM - CPU)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 16, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Memory_Heap_Used{tm_id=~\"$tm_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Heap", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Memory_NonHeap_Used{tm_id=~\"$tm_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Non Heap", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Memory_Direct_MemoryUsed{tm_id=~\"$tm_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Direct", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Memory_Mapped_MemoryUsed{tm_id=~\"$tm_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Mapped", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_Memory_Metaspace_Used{tm_id=~\"$tm_id\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Metaspace", + "type": "timeseries" + } + ], + "title": "Task Manager (JVM - Memory Usage)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 48, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_GarbageCollector_G1_Young_Generation_Time{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Young Generation Time", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_GarbageCollector_G1_Old_Generation_Time{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Old Generation Time", + "type": "timeseries" + } + ], + "title": "Task Manager (JVM - Garbage Collector)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 135, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Flink_Memory_Managed_Used{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Managed", + "type": "timeseries" + } + ], + "title": "Task Manager (Memory - Flink)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 55, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_TotalMemory{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Total Memory", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_TotalMemorySegments{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Total Memory Segments", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 20 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_UsedMemory{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Used Memory", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 20 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_AvailableMemory{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Available Memory", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 20 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_UsedMemorySegments{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Used Memory Segments", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 20 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_AvailableMemorySegments{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Available Memory Segments", + "type": "timeseries" + } + ], + "title": "Task Manager (Memory - Shuffle Netty)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 53, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Is Back Pressured", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 11 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_idleTimeMsPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Idle Time Ms (per Second)", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 11 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_checkpointStartDelayNanos{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Checkpoint Alignment Time", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_checkpointAlignmentTime{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Checkpoint Alignment Time", + "type": "timeseries" + } + ], + "title": "Task Manager (Job Task - Diag)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 139, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_numBuffersInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Local", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Local (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_numBuffersInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Remote", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 144, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Remote (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 145, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBuffersOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers Out", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 146, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers Out (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 0, + "y": 29 + }, + "id": 147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBytesIn{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 4, + "y": 29 + }, + "id": 148, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBytesInPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 8, + "y": 29 + }, + "id": 149, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBytesInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In Local", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 12, + "y": 29 + }, + "id": 150, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBytesInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In Local (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 16, + "y": 29 + }, + "id": 151, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBytesInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In Remote", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 20, + "y": 29 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBytesInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In Remote (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBytesOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes Out", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBytesOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes Out (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 47 + }, + "id": 155, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numRecordsIn{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records In", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 47 + }, + "id": 156, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numRecordsInPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes In (per Second)", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 47 + }, + "id": 157, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numRecordsOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records Out", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 47 + }, + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numRecordsOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Bytes Out (per Second)", + "type": "timeseries" + } + ], + "title": "Task Manager (Job Task - General)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 97, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "In Pool Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Queue Length", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 13 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputExclusiveBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Exclusive Buffers Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_outPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Out Pool Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_outputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Output Queue Length", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputFloatingBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Floating Buffers Usage", + "type": "timeseries" + } + ], + "title": "Task Manager (Job Task - Buffers)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 82, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 4, + "x": 0, + "y": 14 + }, + "id": 84, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_operator_numLateRecordsDropped", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Late Records Dropped", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 4, + "y": 14 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_operator_numRecordsIn{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Records In", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 9, + "y": 14 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_operator_numRecordsInPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Records Out (per Second)", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 14, + "y": 14 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_operator_numRecordsOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Records In", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 19, + "y": 14 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_operator_numRecordsOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Num Records Out (per Second)", + "type": "timeseries" + } + ], + "title": "Task Manager (Job Task - Operator)", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 60, + "panels": [ + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Queue Length", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Output_Buffers_outputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Output Queue Length", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "In Pool Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Output_Buffers_outPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Out Pool Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputFloatingBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Floating Buffers Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputExclusiveBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Exclusive Buffers Usage", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Local", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Local (per Second)", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 33 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Remote", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 33 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Num Buffers In Remote (per Second)", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Num Bytes In Local", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Local (per Second)", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Remote", + "type": "timeseries" + }, + { + "datasource": "${Source}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Remote (per Second)", + "type": "timeseries" + } + ], + "title": "Task Manager (Job Task - Shuffle Netty)", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 30, + "style": "dark", + "tags": [ + "flink", + "task manager", + "job manager", + "apache" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "prometheus" + ], + "value": [ + "prometheus" + ] + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "Source", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(tm_id)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Task Manager", + "multi": true, + "name": "tm_id", + "options": [], + "query": { + "query": "label_values(tm_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "${Source}", + "definition": "label_values(flink_jobmanager_numRegisteredTaskManagers,instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Job Manager", + "multi": true, + "name": "jm_instance", + "options": [], + "query": { + "query": "label_values(flink_jobmanager_numRegisteredTaskManagers,instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "label_values(flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\"},job_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Job Name", + "multi": true, + "name": "job_name", + "options": [], + "query": { + "query": "label_values(flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\"},job_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "${Source}", + "definition": "label_values(flink_taskmanager_job_task_isBackPressured{job_name=~\"$job_name\"},task_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Task Name", + "multi": true, + "name": "task_name", + "options": [], + "query": { + "query": "label_values(flink_taskmanager_job_task_isBackPressured{job_name=~\"$job_name\"},task_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": null, + "description": null, + "error": null, + "filters": [], + "hide": 0, + "label": "Ad Hoc", + "name": "ad_hoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Flink", + "uid": "wKbnD5Gnk", + "version": 1 + } \ No newline at end of file diff --git a/environment/plaintext/prometheus/prometheus.yml b/environment/plaintext/prometheus/prometheus.yml index 464b9d04e..f99379c9d 100644 --- a/environment/plaintext/prometheus/prometheus.yml +++ b/environment/plaintext/prometheus/prometheus.yml @@ -25,6 +25,14 @@ scrape_configs: static_configs: - targets: ["localhost:9090"] + - job_name: 'flink-jobmanager' + static_configs: + - targets: ['jobmanager:9090'] + + - job_name: 'flink-taskmanager' + static_configs: + - targets: ['taskmanager:9090'] + - job_name: "node-exporter" static_configs: diff --git a/flink/flink_app_mode/docker-compose.yml b/flink/flink_app_mode/docker-compose.yml index 7e8c0160c..193e68560 100644 --- a/flink/flink_app_mode/docker-compose.yml +++ b/flink/flink_app_mode/docker-compose.yml @@ -8,7 +8,9 @@ services: volumes: - ${FLINK_JAR_PATH}:/opt/flink/job.jar environment: - - FLINK_PROPERTIES= + - | + FLINK_PROPERTIES= + ${GRAFANA_FLINK} taskmanager: image: flink:${FLINK_TAG} @@ -24,4 +26,76 @@ services: FLINK_PROPERTIES= jobmanager.rpc.address: jobmanager taskmanager.numberOfTaskSlots: 2 - parallelism.default: 2 \ No newline at end of file + parallelism.default: 2 + ${GRAFANA_FLINK} + + prometheus: + image: prom/prometheus:v2.29.2 + hostname: prometheus + container_name: prometheus + profiles: + - "grafana" + ports: + - 9090:9090 + volumes: + - ../../environment/plaintext/prometheus/:/etc/prometheus/ + depends_on: + - node-exporter + - kafka-lag-exporter + - alertmanager + + grafana: + image: grafana/grafana:8.5.27 + hostname: grafana + container_name: grafana + profiles: + - "grafana" + environment: + - "GF_SECURITY_ADMIN_USER=admin" + - "GF_SECURITY_ADMIN_PASSWORD=password" + - "GF_USERS_ALLOW_SIGN_UP=false" + ports: + - 3000:3000 + volumes: + - ../../environment/plaintext/grafana/provisioning/:/etc/grafana/provisioning/ + - ../../environment/plaintext/grafana/config/grafana.ini:/etc/grafana/grafana.ini + depends_on: + - prometheus + + node-exporter: + image: prom/node-exporter:v1.2.2 + hostname: node-exporter + container_name: node-exporter + profiles: + - "grafana" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.ignored-mount-points" + - "^(aufs|proc|nsfs|shm|cgroup|tmpfs|binfmt_misc|debugfs|devpts|fusectl|hugetlbfs|fuse.lxcfs|mqueue|pstore|securityfs|sysfs|autofs|devtmpfs|configfs)" + + kafka-lag-exporter: + image: seglo/kafka-lag-exporter:0.7.1 + hostname: kafka-lag-exporter + container_name: kafka-lag-exporter + profiles: + - "grafana" + restart: always + ports: + - 9998:9998 + volumes: + - ../../environment/plaintext/kafka-lag-exporter/application.conf:/opt/docker/conf/application.conf + - ../../environment/plaintext/kafka-lag-exporter/logback.xml:/opt/docker/conf/logback.xml + + alertmanager: + image: prom/alertmanager:latest + hostname: alertmanager + container_name: alertmanager + profiles: + - "grafana" + ports: + - 9093:9093 \ No newline at end of file diff --git a/flink/flink_app_mode/start.sh b/flink/flink_app_mode/start.sh index 1a894d841..336ebc2c2 100755 --- a/flink/flink_app_mode/start.sh +++ b/flink/flink_app_mode/start.sh @@ -11,6 +11,16 @@ then exit fi +profile_grafana_command="" +if [ -z "$ENABLE_JMX_GRAFANA" ] +then + log "🛑 Grafana is disabled" +else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 +fi + source ${DIR}/../../scripts/flink_download_connectors.sh -docker-compose up -d \ No newline at end of file +docker compose ${profile_grafana_command} up -d \ No newline at end of file diff --git a/flink/flink_app_mode/stop.sh b/flink/flink_app_mode/stop.sh index eb1a7c66c..b27c220ba 100755 --- a/flink/flink_app_mode/stop.sh +++ b/flink/flink_app_mode/stop.sh @@ -3,5 +3,15 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/../../scripts/utils.sh -docker-compose down -v --remove-orphans +profile_grafana_command="" +if [ -z "$ENABLE_JMX_GRAFANA" ] +then + log "🛑 Grafana is disabled" +else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 +fi + +docker-compose ${profile_grafana_command} down -v --remove-orphans stop_all "$DIR" diff --git a/flink/flink_session_mode/docker-compose.yml b/flink/flink_session_mode/docker-compose.yml index 5075dcb66..7be94653d 100644 --- a/flink/flink_session_mode/docker-compose.yml +++ b/flink/flink_session_mode/docker-compose.yml @@ -21,4 +21,77 @@ services: - | FLINK_PROPERTIES= jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 \ No newline at end of file + taskmanager.numberOfTaskSlots: 2 + ${GRAFANA_FLINK} + + prometheus: + image: prom/prometheus:v2.29.2 + hostname: prometheus + container_name: prometheus + profiles: + - "grafana" + ports: + - 9090:9090 + volumes: + - ../../environment/plaintext/prometheus/:/etc/prometheus/ + + depends_on: + - node-exporter + - kafka-lag-exporter + - alertmanager + + grafana: + image: grafana/grafana:8.5.27 + hostname: grafana + container_name: grafana + profiles: + - "grafana" + environment: + - "GF_SECURITY_ADMIN_USER=admin" + - "GF_SECURITY_ADMIN_PASSWORD=password" + - "GF_USERS_ALLOW_SIGN_UP=false" + ports: + - 3000:3000 + volumes: + - ../../environment/plaintext/grafana/provisioning/:/etc/grafana/provisioning/ + - ../../environment/plaintext/grafana/config/grafana.ini:/etc/grafana/grafana.ini + depends_on: + - prometheus + + node-exporter: + image: prom/node-exporter:v1.2.2 + hostname: node-exporter + container_name: node-exporter + profiles: + - "grafana" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.ignored-mount-points" + - "^(aufs|proc|nsfs|shm|cgroup|tmpfs|binfmt_misc|debugfs|devpts|fusectl|hugetlbfs|fuse.lxcfs|mqueue|pstore|securityfs|sysfs|autofs|devtmpfs|configfs)" + + kafka-lag-exporter: + image: seglo/kafka-lag-exporter:0.7.1 + hostname: kafka-lag-exporter + container_name: kafka-lag-exporter + profiles: + - "grafana" + restart: always + ports: + - 9998:9998 + volumes: + - ../../environment/plaintext/kafka-lag-exporter/application.conf:/opt/docker/conf/application.conf + - ../../environment/plaintext/kafka-lag-exporter/logback.xml:/opt/docker/conf/logback.xml + + alertmanager: + image: prom/alertmanager:latest + hostname: alertmanager + container_name: alertmanager + profiles: + - "grafana" + ports: + - 9093:9093 \ No newline at end of file diff --git a/flink/flink_session_mode/start.sh b/flink/flink_session_mode/start.sh index 3f6a696f5..f3e104a4d 100755 --- a/flink/flink_session_mode/start.sh +++ b/flink/flink_session_mode/start.sh @@ -4,7 +4,15 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # loading env variables source ${DIR}/../../scripts/utils.sh +profile_grafana_command="" + if [ -z "$ENABLE_JMX_GRAFANA" ] + then + log "🛑 Grafana is disabled" + else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 + fi source ${DIR}/../../scripts/flink_download_connectors.sh - -docker-compose up -d \ No newline at end of file +docker compose ${profile_grafana_command} up -d \ No newline at end of file diff --git a/flink/flink_session_mode/stop.sh b/flink/flink_session_mode/stop.sh index 7962d4633..a86c77b67 100755 --- a/flink/flink_session_mode/stop.sh +++ b/flink/flink_session_mode/stop.sh @@ -1,9 +1,19 @@ #!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -echo "Pops ${DIR}" + source ${DIR}/../../scripts/utils.sh -docker-compose down -v --remove-orphans +profile_grafana_command="" +if [ -z "$ENABLE_JMX_GRAFANA" ] +then + log "🛑 Grafana is disabled" +else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 +fi + +docker-compose ${profile_grafana_command} down -v --remove-orphans stop_all "$DIR" \ No newline at end of file diff --git a/flink/flink_session_sql_mode/docker-compose.yml b/flink/flink_session_sql_mode/docker-compose.yml index 8cfda4dfd..f2f6d8fee 100644 --- a/flink/flink_session_sql_mode/docker-compose.yml +++ b/flink/flink_session_sql_mode/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: # Flink JobManager jobmanager: @@ -14,6 +12,7 @@ services: - | FLINK_PROPERTIES= jobmanager.rpc.address: jobmanager + ${GRAFANA_FLINK} # Flink TaskManager taskmanager: @@ -30,6 +29,7 @@ services: jobmanager.rpc.address: jobmanager rest.address: jobmanager taskmanager.numberOfTaskSlots: 4 + ${GRAFANA_FLINK} # Flink SQL Client sql-client: @@ -48,4 +48,75 @@ services: FLINK_PROPERTIES= jobmanager.rpc.address: jobmanager rest.address: jobmanager - deployment.gateway-address: jobmanager \ No newline at end of file + deployment.gateway-address: jobmanager + + prometheus: + image: prom/prometheus:v2.29.2 + hostname: prometheus + container_name: prometheus + profiles: + - "grafana" + ports: + - 9090:9090 + volumes: + - ../../environment/plaintext/prometheus/:/etc/prometheus/ + depends_on: + - node-exporter + - kafka-lag-exporter + - alertmanager + + grafana: + image: grafana/grafana:8.5.27 + hostname: grafana + container_name: grafana + profiles: + - "grafana" + environment: + - "GF_SECURITY_ADMIN_USER=admin" + - "GF_SECURITY_ADMIN_PASSWORD=password" + - "GF_USERS_ALLOW_SIGN_UP=false" + ports: + - 3000:3000 + volumes: + - ../../environment/plaintext/grafana/provisioning/:/etc/grafana/provisioning/ + - ../../environment/plaintext/grafana/config/grafana.ini:/etc/grafana/grafana.ini + depends_on: + - prometheus + + node-exporter: + image: prom/node-exporter:v1.2.2 + hostname: node-exporter + container_name: node-exporter + profiles: + - "grafana" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.ignored-mount-points" + - "^(aufs|proc|nsfs|shm|cgroup|tmpfs|binfmt_misc|debugfs|devpts|fusectl|hugetlbfs|fuse.lxcfs|mqueue|pstore|securityfs|sysfs|autofs|devtmpfs|configfs)" + + kafka-lag-exporter: + image: seglo/kafka-lag-exporter:0.7.1 + hostname: kafka-lag-exporter + container_name: kafka-lag-exporter + profiles: + - "grafana" + restart: always + ports: + - 9998:9998 + volumes: + - ../../environment/plaintext/kafka-lag-exporter/application.conf:/opt/docker/conf/application.conf + - ../../environment/plaintext/kafka-lag-exporter/logback.xml:/opt/docker/conf/logback.xml + + alertmanager: + image: prom/alertmanager:latest + hostname: alertmanager + container_name: alertmanager + profiles: + - "grafana" + ports: + - 9093:9093 \ No newline at end of file diff --git a/flink/flink_session_sql_mode/start.sh b/flink/flink_session_sql_mode/start.sh index 8d514a733..1f5843f0f 100755 --- a/flink/flink_session_sql_mode/start.sh +++ b/flink/flink_session_sql_mode/start.sh @@ -6,4 +6,14 @@ source ${DIR}/../../scripts/utils.sh source ${DIR}/../../scripts/flink_download_connectors.sh -docker-compose up -d \ No newline at end of file +profile_grafana_command="" +if [ -z "$ENABLE_JMX_GRAFANA" ] +then + log "🛑 Grafana is disabled" +else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 +fi + +docker compose ${profile_grafana_command} up -d \ No newline at end of file diff --git a/flink/flink_session_sql_mode/stop.sh b/flink/flink_session_sql_mode/stop.sh index 92b92d34a..44c0c049e 100755 --- a/flink/flink_session_sql_mode/stop.sh +++ b/flink/flink_session_sql_mode/stop.sh @@ -3,6 +3,16 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/../../scripts/utils.sh -docker-compose down -v --remove-orphans +profile_grafana_command="" +if [ -z "$ENABLE_JMX_GRAFANA" ] +then + log "🛑 Grafana is disabled" +else + log "📊 Grafana is enabled" + profile_grafana_command="--profile grafana" + playground state set flags.ENABLE_JMX_GRAFANA 1 +fi + +docker-compose ${profile_grafana_command} down -v --remove-orphans stop_all "$DIR" \ No newline at end of file diff --git a/scripts/flink_download_connectors.sh b/scripts/flink_download_connectors.sh index bf5cde2f7..034041921 100755 --- a/scripts/flink_download_connectors.sh +++ b/scripts/flink_download_connectors.sh @@ -1,139 +1,145 @@ #!/bin/bash -log() { - echo "$@" +logging() { + log "$@" } - -log "❗ Do you need to install any Flink connectors? (yes/no)" -read need_connectors - -if [[ "$need_connectors" != "yes" ]]; then - log "Skipping connector installation steps." +# Allow users to provide URL for what to download +if [ -n "$FLINK_URL" ]; then + logging "❗ Downloading connectors from $FLINK_URL, full command executed will be 'wget -P /opt/flink/lib $FLINK_URL' " + flink_connectors="wget -P /opt/flink/lib $FLINK_URL && " + export flink_connectors else - log "❗ Is it one connector or multiple connectors? (one/multiple)" - read connector_count + logging "❗ Do you need to install any Flink connectors? (yes/no)" + read need_connectors - if [[ "$connector_count" == "one" ]]; then - log "Please type one word that's related to the connector name:" - read keyword - keywords=("$keyword") + if [[ "$need_connectors" != "yes" ]]; then + logging "Skipping connector installation steps." else - log "Please type multiple words related to the connector names (separated by spaces):" - read -a keywords - fi + logging "❗ Is it one connector or multiple connectors? (one/multiple)" + read connector_count - # Function to fetch connector list - get_connectors() { - curl -s https://repo.maven.apache.org/maven2/org/apache/flink/ | awk -F '"' '/href/ {print $2}' | sed 's:/$::' | sort -u - } + if [[ "$connector_count" == "one" ]]; then + logging "Please type one word that's related to the connector name:" + read keyword + keywords=("$keyword") + else + logging "Please type multiple words related to the connector names (separated by spaces):" + read -a keywords + fi - connector_list=$(get_connectors) - selected_connectors=() + # Function to fetch connector list + get_connectors() { + curl -s https://repo.maven.apache.org/maven2/org/apache/flink/ | awk -F '"' '/href/ {print $2}' | sed 's:/$::' | sort -u + } - # Loop through each keyword - for keyword in "${keywords[@]}"; do - matches=$(echo "$connector_list" | grep -i "$keyword") - if [[ -z "$matches" ]]; then - log "No connectors found matching '$keyword'." - continue - fi - log "Connectors matching '$keyword':" - i=1 - match_list=() - while IFS= read -r line; do - echo "$i) $line" - match_list+=("$line") - ((i++)) - done <<< "$matches" + connector_list=$(get_connectors) + selected_connectors=() - log "Please select connectors by numbers (separated by spaces):" - read selection - IFS=' ' read -r -a selections <<< "$selection" - for sel in "${selections[@]}"; do - if [[ $sel -le ${#match_list[@]} && $sel -ge 1 ]]; then - selected_connector=${match_list[$((sel - 1))]} - selected_connectors+=("$selected_connector") - log "Selected connector: $selected_connector" - else - log "Invalid selection: $sel" + # Loop through each keyword + for keyword in "${keywords[@]}"; do + matches=$(echo "$connector_list" | grep -i "$keyword") + if [[ -z "$matches" ]]; then + logging "No connectors found matching '$keyword'." + continue fi + logging "Connectors matching '$keyword':" + i=1 + match_list=() + while IFS= read -r line; do + echo "$i) $line" + match_list+=("$line") + ((i++)) + done <<< "$matches" + + logging "Please select connectors by numbers (separated by spaces):" + read selection + IFS=' ' read -r -a selections <<< "$selection" + for sel in "${selections[@]}"; do + if [[ $sel -le ${#match_list[@]} && $sel -ge 1 ]]; then + selected_connector=${match_list[$((sel - 1))]} + selected_connectors+=("$selected_connector") + logging "Selected connector: $selected_connector" + else + logging "Invalid selection: $sel" + fi + done done - done - - connector_urls=() - - # Loop through each selected connector - for connector in "${selected_connectors[@]}"; do - log "Fetching versions for connector: $connector" - connector_url="https://repo.maven.apache.org/maven2/org/apache/flink/$connector/" - versions=$(curl -s "$connector_url" | awk -F '"' '/href/ {print $2}' | sed 's:/$::' | sort -u) - if [[ -z "$versions" ]]; then - log "No versions found for connector '$connector'." - continue - fi - log "Available versions for $connector:" - i=1 - version_list=() - while IFS= read -r version; do - echo "$i) $version" - version_list+=("$version") - ((i++)) - done <<< "$versions" - - log "Please select a version by number for connector $connector:" - read version_selection - if [[ $version_selection -le ${#version_list[@]} && $version_selection -ge 1 ]]; then - selected_version=${version_list[$((version_selection - 1))]} - else - log "Invalid selection. Skipping connector $connector." - continue - fi - # Fetch jar files for the selected version - jar_url="$connector_url$selected_version/" - log "Fetching jar files for $connector version $selected_version" - jar_files=$(curl -s "$jar_url" | awk -F '"' '/href/ {print $2}' | grep -E '\.jar$' | sort -u) - if [[ -z "$jar_files" ]]; then - log "No jar files found for $connector version $selected_version." - continue - fi - log "Available jar files for $connector version $selected_version:" - i=1 - jar_list=() - while IFS= read -r jar; do - echo "$i) $jar" - jar_list+=("$jar") - ((i++)) - done <<< "$jar_files" + connector_urls=() - log "❗ Please select jar files by numbers (separated by spaces), or type ⭐'complete'⭐ to finish selection:" - read jar_selection - if [[ "$jar_selection" == "complete" ]]; then - continue - fi - IFS=' ' read -r -a jar_selections <<< "$jar_selection" - for js in "${jar_selections[@]}"; do - if [[ $js -le ${#jar_list[@]} && $js -ge 1 ]]; then - selected_jar=${jar_list[$((js - 1))]} - full_url="$jar_url$selected_jar" - connector_urls+=("$full_url") - log "Selected jar file: $selected_jar" + # Loop through each selected connector + for connector in "${selected_connectors[@]}"; do + logging "Fetching versions for connector: $connector" + connector_url="https://repo.maven.apache.org/maven2/org/apache/flink/$connector/" + versions=$(curl -s "$connector_url" | awk -F '"' '/href/ {print $2}' | sed 's:/$::' | sort -u) + if [[ -z "$versions" ]]; then + logging "No versions found for connector '$connector'." + continue + fi + logging "Available versions for $connector:" + i=1 + version_list=() + while IFS= read -r version; do + echo "$i) $version" + version_list+=("$version") + ((i++)) + done <<< "$versions" + + logging "Please select a version by number for connector $connector:" + read version_selection + if [[ $version_selection -le ${#version_list[@]} && $version_selection -ge 1 ]]; then + selected_version=${version_list[$((version_selection - 1))]} else - log "Invalid selection: $js" + logging "Invalid selection. Skipping connector $connector." + continue + fi + + # Fetch jar files for the selected version + jar_url="$connector_url$selected_version/" + logging "Fetching jar files for $connector version $selected_version" + jar_files=$(curl -s "$jar_url" | awk -F '"' '/href/ {print $2}' | grep -E '\.jar$' | sort -u) + if [[ -z "$jar_files" ]]; then + logging "No jar files found for $connector version $selected_version." + continue + fi + logging "Available jar files for $connector version $selected_version:" + i=1 + jar_list=() + while IFS= read -r jar; do + echo "$i) $jar" + jar_list+=("$jar") + ((i++)) + done <<< "$jar_files" + + logging "❗ Please select jar files by numbers (separated by spaces), or type 'complete' to finish selection:" + read jar_selection + if [[ "$jar_selection" == "complete" ]]; then + continue fi + IFS=' ' read -r -a jar_selections <<< "$jar_selection" + for js in "${jar_selections[@]}"; do + if [[ $js -le ${#jar_list[@]} && $js -ge 1 ]]; then + selected_jar=${jar_list[$((js - 1))]} + full_url="$jar_url$selected_jar" + connector_urls+=("$full_url") + logging "Selected jar file: $selected_jar" + else + logging "Invalid selection: $js" + fi + done done - done - # Set the environment variable 'connectors' - if [ -z "${connector_urls[*]}" ]; then - flink_connectors=" " - else - flink_connectors="wget -P /opt/flink/lib ${connector_urls[*]} && " - fi - export flink_connectors + # Set the environment variable 'connectors' + if [ -z "${connector_urls[*]}" ]; then + flink_connectors=" " + else + flink_connectors="wget -P /opt/flink/lib ${connector_urls[*]} && " + fi + export flink_connectors - log "Environment variable 'flink_connectors' set with the following URLs which will be downloaded for the containers:" - log "$flink_connectors" + logging "Environment variable 'flink_connectors' set with the following URLs which will be downloaded for the containers:" + logging "$flink_connectors" + fi fi -log "Script completed." \ No newline at end of file +logging "Script completed." \ No newline at end of file diff --git a/scripts/utils.sh b/scripts/utils.sh index 51460a7ba..3f271dd12 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -125,6 +125,7 @@ then export GRAFANA_AGENT_CONSUMER="" export GRAFANA_AGENT_SR="" export GRAFANA_AGENT_KSQLDB="" + export GRAFANA_FLINK="" else export GRAFANA_AGENT_ZK="-javaagent:/usr/share/jmx_exporter/pyroscope-0.11.2.jar -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=1234:/usr/share/jmx_exporter/zookeeper.yml" export GRAFANA_AGENT_BROKER="-javaagent:/usr/share/jmx_exporter/pyroscope-0.11.2.jar -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=1234:/usr/share/jmx_exporter/kafka_broker.yml" @@ -133,6 +134,8 @@ else export GRAFANA_AGENT_CONSUMER="-javaagent:/usr/share/jmx_exporter/pyroscope-0.11.2.jar -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=1234:/usr/share/jmx_exporter/kafka-consumer.yml" export GRAFANA_AGENT_SR="-javaagent:/usr/share/jmx_exporter/pyroscope-0.11.2.jar -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=1234:/usr/share/jmx_exporter/confluent_schemaregistry.yml" export GRAFANA_AGENT_KSQLDB="-javaagent:/usr/share/jmx_exporter/pyroscope-0.11.2.jar -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent-0.20.0.jar=1234:/usr/share/jmx_exporter/confluent_ksql.yml" + export GRAFANA_FLINK="metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.port: 9090" fi # Migrate SimpleAclAuthorizer to AclAuthorizer #1276