Prometheus
https://github.com/narethim/tools/blob/master/setup-prometheus-node-exporter-arm64.md
https://grafana.com/docs/grafana-cloud/quickstart/noagent_linuxnode/
docker run -d --restart=always --net mqtt \
--name=prometheus \
-p 9099:9090 \
--add-host=host.docker.internal:host-gateway \
-v ~/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
prom/prometheus
Linux Exporter
https://linuxhit.com/prometheus-node-exporter-on-raspberry-pi-how-to-install/
https://stackoverflow.com/questions/31324981/how-to-access-host-port-from-docker-container
wget https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-arm64.tar.gz tar -xvzf node_exporter-0.18.1.linux-arm64.tar.gz sudo cp node_exporter-0.18.1.linux-arm64/node_exporter /usr/local/bin sudo chmod +x /usr/local/bin/node_exporter sudo useradd -m -s /bin/bash node_exporter sudo mkdir /var/lib/node_exporter sudo chown -R node_exporter:node_exporter /var/lib/node_exporter sudo nano /etc/systemd/system/node_exporter.service
[Unit] Description=Node Exporter [Service] # Provide a text file location for https://github.com/fahlke/raspberrypi_exporter data with the # --collector.textfile.directory parameter. ExecStart=/usr/local/bin/node_exporter --collector.textfile.directory /var/lib/node_exporter/textfile_collector [Install] WantedBy=multi-user.target
sudo systemctl daemon-reload sudo systemctl enable node_exporter.service sudo systemctl start node_exporter.service
Prometheus configuration
- job_name: linux
honor_timestamps: true
scrape_interval: 60s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
follow_redirects: true
enable_http2: true
static_configs:
- targets:
- host.docker.internal:9100
Grafana Dashboard
In Grafana, click Dashboards in the left side menu (looks like four boxes) and from the menu that appears select Manage.
Click Import. Enter the ID number 10180 for the Linux dashboard, and click Load.
OR Import the following JSON. This has had a minor adjustment to make the CPU show.
{"__inputs":[{"name":"DS_PROMETHEUS","label":"Prometheus","description":"","type":"datasource","pluginId":"prometheus","pluginName":"Prometheus"}],"__elements":{},"__requires":[{"type":"panel","id":"gauge","name":"Gauge","version":""},{"type":"grafana","id":"grafana","name":"Grafana","version":"9.2.1"},{"type":"panel","id":"graph","name":"Graph (old)","version":""},{"type":"datasource","id":"prometheus","name":"Prometheus","version":"1.0.0"},{"type":"panel","id":"stat","name":"Stat","version":""},{"type":"panel","id":"table-old","name":"Table (old)","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Basic overview of linux host metrics, based on node_exporter","editable":true,"fiscalYearStartMonth":0,"gnetId":10180,"graphTooltip":1,"id":null,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":42,"panels":[],"targets":[{"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"refId":"A"}],"title":"Host Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Time since last boot","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":4,"w":2,"x":0,"y":1},"id":6,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_time_seconds{instance=~\"$host\"} - node_boot_time_seconds{instance=~\"$host\"}","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Uptime | $host","type":"stat"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Number of processors","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":2,"x":2,"y":1},"id":2,"interval":"","links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"count(count(node_cpu_seconds_total{instance=~\"$host\"}) by (cpu))","format":"time_series","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Processors","type":"stat"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Amount of memory","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":0,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":4,"w":2,"x":4,"y":1},"id":4,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_memory_MemTotal_bytes{instance=~\"$host\"}","format":"time_series","instant":true,"intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"RAM","type":"stat"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":100,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":60},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":6,"y":1},"id":14,"links":[],"options":{"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"editorMode":"code","expr":"sum by (mode) (irate(node_cpu_seconds_total{instance=~\"$host\",mode=~\"idle\"}[5m]))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"__auto","range":true,"refId":"B"}],"title":"CPU Load","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#299c46","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":0.6},{"color":"#d44a3a","value":0.8}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":9,"y":1},"id":35,"links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"1 - node_memory_MemFree_bytes{instance=~\"$host\"} / node_memory_MemTotal_bytes{instance=~\"$host\"}","format":"time_series","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Memory Use","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Free diskspace","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"decimals":1,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#299c46","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":0.6},{"color":"#d44a3a","value":0.8}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":4,"w":3,"x":12,"y":1},"id":8,"interval":"","links":[],"maxDataPoints":100,"options":{"orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"1 - (sum(node_filesystem_free_bytes{instance=~\"$host\"}) / sum(node_filesystem_size_bytes{instance=~\"$host\"}))","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Disk Free (Total)","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Network traffic in the last hour","fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":4,"w":2,"x":15,"y":1},"id":10,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"sum(increase(node_network_receive_bytes_total{instance=~\"$host\"}[24h]))","format":"time_series","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Net IN (24h)","type":"stat"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"description":"Network traffic in the last hour","fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"decbytes"},"overrides":[]},"gridPos":{"h":4,"w":2,"x":17,"y":1},"id":12,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"9.2.1","repeat":"host","repeatDirection":"v","targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"sum(increase(node_network_transmit_bytes_total{instance=~\"$host\"}[24h]))","format":"time_series","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Net OUT (24h)","type":"stat"},{"columns":[],"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fontSize":"100%","gridPos":{"h":4,"w":5,"x":19,"y":1},"id":37,"links":[],"repeat":"host","repeatDirection":"v","scroll":true,"showHeader":true,"sort":{"col":0,"desc":true},"styles":[{"alias":"Available","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":1,"mappingType":1,"pattern":"Value","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Mount","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"mountpoint","thresholds":[],"type":"string","unit":"short"},{"alias":"Type","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"fstype","thresholds":[],"type":"string","unit":"short"},{"alias":"","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"decimals":2,"pattern":"/.*/","thresholds":[],"type":"hidden","unit":"short"}],"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_filesystem_free_bytes{fstype!~\"(tmpfs|rootfs).*\",instance=~\"$host\"}","format":"table","instant":true,"intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Disk (Free)","transform":"table","type":"table-old"},{"collapsed":false,"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":26,"panels":[],"targets":[{"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"refId":"A"}],"title":"CPU Details","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":24,"x":0,"y":6},"hiddenSeries":false,"id":44,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":true,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"sum by (mode) (irate(node_cpu_seconds_total{instance=~\"$host\"}[5m]))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"{{mode}}","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"CPU Load | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":1,"format":"short","logBase":1,"min":"0","show":true},{"format":"short","logBase":1,"show":false}],"yaxis":{"align":false}},{"collapsed":true,"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":24,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":1,"fillGradient":0,"gridPos":{"h":6,"w":24,"x":0,"y":13},"hiddenSeries":false,"id":16,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":true,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_memory_MemFree_bytes{instance=~\"$host\"} > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Free","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_memory_MemTotal_bytes{instance=~\"$host\"} > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Total","refId":"B"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_memory_MemAvailable_bytes{instance=~\"$host\"} > 0","format":"time_series","intervalFactor":1,"legendFormat":"Available","refId":"C"}],"thresholds":[],"timeRegions":[],"title":"Memory | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":0,"format":"bytes","label":"","logBase":1,"min":"0","show":true},{"format":"short","logBase":1,"show":false}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"refId":"A"}],"title":"Memory Details","type":"row"},{"collapsed":true,"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"gridPos":{"h":1,"w":24,"x":0,"y":13},"id":28,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":2,"fillGradient":0,"gridPos":{"h":6,"w":14,"x":0,"y":14},"hiddenSeries":false,"id":18,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":true,"min":true,"rightSide":true,"show":true,"sort":"min","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"irate(node_network_receive_bytes_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m]) > 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"IN ({{device}})","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"- irate(node_network_transmit_bytes_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m]) < 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"OUT ({{device}})","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Network Traffic | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"show":true},{"format":"short","label":"","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":2,"fillGradient":0,"gridPos":{"h":6,"w":10,"x":14,"y":14},"hiddenSeries":false,"id":43,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":true,"min":true,"rightSide":true,"show":true,"sort":"min","sortDesc":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"irate(node_network_receive_errs_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m]) + irate(node_network_receive_drop_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m])","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Err/Drop IN ({{device}})","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"- (irate(node_network_transmit_errs_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m]) + irate(node_network_transmit_drop_total{instance=~\"$host\",device=~\"(?i)^(ens|eth).+$\"}[5m]))","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Err/Drop OUT ({{device}})","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Network Traffic | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":1,"format":"pps","label":"","logBase":1,"show":true},{"format":"short","label":"","logBase":1,"show":true}],"yaxis":{"align":false}}],"targets":[{"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"refId":"A"}],"title":"Network Details","type":"row"},{"collapsed":false,"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"gridPos":{"h":1,"w":24,"x":0,"y":14},"id":22,"panels":[],"targets":[{"datasource":{"type":"influxdb","uid":"QOSMrINVk"},"refId":"A"}],"title":"Disk Details | $host","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":1,"fillGradient":0,"gridPos":{"h":7,"w":9,"x":0,"y":15},"hiddenSeries":false,"id":40,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"node_filesystem_free_bytes{instance=~\"$host\",fstype!~\"(tmpfs|rootfs)\"}","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"{{mountpoint}}","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Disk (Free) | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":9,"y":15},"hiddenSeries":false,"id":30,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":true,"min":true,"rightSide":true,"show":true,"sort":"min","sortDesc":false,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"irate(node_disk_read_bytes_total{instance=~\"$host\"}[5m]) > 0","format":"time_series","hide":false,"interval":"","intervalFactor":1,"legendFormat":"Read ({{device}})","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"- irate(node_disk_written_bytes_total{instance=~\"$host\"}[5m]) < 0","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Write ({{device}})","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Disk Activity | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"fill":1,"fillGradient":0,"gridPos":{"h":7,"w":7,"x":17,"y":15},"hiddenSeries":false,"id":32,"legend":{"alignAsTable":true,"avg":false,"current":false,"max":true,"min":true,"rightSide":true,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"9.2.1","pointradius":5,"points":false,"renderer":"flot","repeat":"host","repeatDirection":"v","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"irate(node_disk_read_time_seconds_total{instance=~\"$host\"}[5m]) > 0","format":"time_series","intervalFactor":1,"legendFormat":"Read ({{device}})","refId":"A"},{"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"expr":"- irate(node_disk_write_time_seconds_total{instance=~\"$host\"}[5m]) < 0","format":"time_series","intervalFactor":1,"legendFormat":"Write ({{device}})","refId":"B"}],"thresholds":[],"timeRegions":[],"title":"Disk IO | $host","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"s","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}}],"refresh":false,"schemaVersion":37,"style":"dark","tags":["linux","node-exporter","ops"],"templating":{"list":[{"current":{},"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"definition":"label_values(node_time_seconds{job=\"$job\"},instance)","hide":0,"includeAll":false,"label":"Host","multi":true,"name":"host","options":[],"query":{"query":"label_values(node_time_seconds{job=\"$job\"},instance)","refId":"Prometheus-host-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{},"datasource":{"type":"prometheus","uid":"${DS_PROMETHEUS}"},"definition":"label_values(node_boot_time_seconds,job)","hide":0,"includeAll":false,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(node_boot_time_seconds,job)","refId":"Prometheus-job-Variable-Query"},"refresh":1,"regex":"/(.*)/","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Linux Hosts Metrics | Base","uid":"ov0oEgdik","version":2,"weekStart":""}
Node-RED
[{"id":"d1f3d002.63ee48","type":"subflow","name":"rbe 5m","info":"","category":"","in":[{"x":60,"y":120,"wires":[{"id":"dd19c0de.9bebd"}]}],"out":[{"x":420,"y":160,"wires":[{"id":"29eae4ad.d68f2c","port":0}]}],"env":[],"meta":{},"color":"#DDAA99"},{"id":"dd19c0de.9bebd","type":"rbe","z":"d1f3d002.63ee48","name":"","func":"rbe","gap":"","start":"","inout":"out","property":"payload","x":190,"y":120,"wires":[["29eae4ad.d68f2c"]]},{"id":"8b22fc2a.83d07","type":"inject","z":"d1f3d002.63ee48","name":"","props":[{"p":"reset","v":"1","vt":"str"}],"repeat":"300","crontab":"","once":false,"onceDelay":0.1,"topic":"","x":200,"y":80,"wires":[["dd19c0de.9bebd"]]},{"id":"29eae4ad.d68f2c","type":"switch","z":"d1f3d002.63ee48","name":"","property":"reset","propertyType":"msg","rules":[{"t":"null"}],"checkall":"true","repair":false,"outputs":1,"x":280,"y":160,"wires":[[]]},{"id":"82572bd7fa2754e2","type":"link out","z":"f6f2187d.f17ca8","name":"link out 9","mode":"link","links":["058f796bc37b7250","56f925dd66127bbb"],"x":265,"y":140,"wires":[]},{"id":"7fddbf716a47caa3","type":"mqtt in","z":"f6f2187d.f17ca8","name":"","topic":"#","qos":"0","datatype":"auto","broker":"61c70212c61507f5","nl":false,"rap":false,"inputs":0,"x":130,"y":140,"wires":[["82572bd7fa2754e2","f04495f04028c894"]]},{"id":"873f73b8bd187f78","type":"subflow:d1f3d002.63ee48","z":"f6f2187d.f17ca8","name":"","x":360,"y":260,"wires":[["5b24a3ea71452cd8","a2a978223a03ad4b","f656e5b024503c03"]]},{"id":"058f796bc37b7250","type":"link in","z":"f6f2187d.f17ca8","name":"link in 1","links":["190658415505c339","82572bd7fa2754e2"],"x":215,"y":260,"wires":[["873f73b8bd187f78"]]},{"id":"f04495f04028c894","type":"debug","z":"f6f2187d.f17ca8","name":"debug 8","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":290,"y":80,"wires":[]},{"id":"f656e5b024503c03","type":"function","z":"f6f2187d.f17ca8","name":"Metrics List","func":"\n\n\nvar metrics = {};\n\nmetrics.timestamp = new Date().getTime();\nmetrics.topic = msg.topic;\nmetrics.payload = msg.payload;\n\nvar mname = msg.topic.replace(/\\//g,\"_\")\n\n\nflow.set(\"metrics.\" + mname, metrics);\n\n\nreturn msg;","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":410,"y":340,"wires":[[]]},{"id":"1fdeb63853f6378a","type":"http in","z":"f6f2187d.f17ca8","name":"","url":"/metrics","method":"get","upload":false,"swaggerDoc":"","x":190,"y":580,"wires":[["0df212d8de0af553"]]},{"id":"78bcb731b3070bde","type":"http in","z":"f6f2187d.f17ca8","name":"","url":"/metrics/:vargroup","method":"get","upload":false,"swaggerDoc":"","x":220,"y":660,"wires":[["a9bfbf0b470f5412"]]},{"id":"a9bfbf0b470f5412","type":"function","z":"f6f2187d.f17ca8","name":"Metrics","func":"\nvar metrics = flow.get(\"metrics\")||{};\n\n\nvar net = global.get(\"config.networkId\") || \"local\";\n\nvar oot = \"\";\n//oot = 'stat_state {network=\"local\", node=\"mynodered1\", device=\"prometheus\", vargroup=\"stat\", varkey=\"state\"} 1\\n';\n\nfor (var m in metrics) {\n\n var tops = metrics[m].topic.split(\"/\");\n \n if (isNaN(metrics[m].payload)) { continue; }\n if (tops[3] != msg.req.params.vargroup) { continue; } \n\n \n oot += tops[3] + '_' + tops[4] + '{network=\"' + net + '\",node=\"' + tops[1] + '\",device=\"' + tops[2] + '\",vargroup=\"' + tops[3] + '\",varkey=\"' + tops[4] + '\"} ' + metrics[m].payload + \"\\n\";\n\n}\n\n\nmsg.payload = oot;\n\nreturn msg;\n\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":520,"y":660,"wires":[["2bf9db2387e3c2c3","011f1ef5757fc34b"]]},{"id":"0df212d8de0af553","type":"function","z":"f6f2187d.f17ca8","name":"Metrics","func":"\n//temperature{ node = \"mynode\", topic = \"zigbee2mqtt_0x00157d00032b1234\" } { { payload } }\n\nvar metrics = flow.get(\"metrics\")||{};\n//flow.set(\"metrics\",{});\n\nvar net = global.get(\"config.networkId\") || \"local\";\n\nvar oot = \"\";\n//oot = 'stat_state {network=\"local\", node=\"mynodered1\", device=\"prometheus\", vargroup=\"stat\", varkey=\"state\"} 1\\n';\n\nfor (var m in metrics) {\n\n if (isNaN(metrics[m].payload)) { continue; }\n \n var tops = metrics[m].topic.split(\"/\");\n\n //if (tops[3] != \"dat\") { continue; } \n\n oot += tops[3] + '_' + tops[4] + '{network=\"' + net + '\",node=\"' + tops[1] + '\",device=\"' + tops[2] + '\",vargroup=\"' + tops[3] + '\",varkey=\"' + tops[4] + '\"} ' + metrics[m].payload + \"\\n\";\n\n}\n\n\nmsg.payload = oot;\n\nreturn msg;\n\n","outputs":1,"noerr":0,"initialize":"","finalize":"","libs":[],"x":520,"y":580,"wires":[["2bf9db2387e3c2c3","011f1ef5757fc34b"]]},{"id":"011f1ef5757fc34b","type":"debug","z":"f6f2187d.f17ca8","name":"debug 16","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":740,"y":580,"wires":[]},{"id":"2bf9db2387e3c2c3","type":"http response","z":"f6f2187d.f17ca8","name":"","statusCode":"","headers":{},"x":730,"y":620,"wires":[]},{"id":"61c70212c61507f5","type":"mqtt-broker","name":"","broker":"mqtt","port":"1883","clientid":"","autoConnect":true,"usetls":false,"protocolVersion":"4","keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","birthMsg":{},"closeTopic":"","closeQos":"0","closePayload":"","closeMsg":{},"willTopic":"","willQos":"0","willPayload":"","willMsg":{},"userProps":"","sessionExpiry":""}]
Prometheus Config
- job_name: dat
honor_timestamps: true
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics/dat
scheme: http
follow_redirects: true
enable_http2: true
static_configs:
- targets:
- mynodered1:1880
- job_name: stat
honor_timestamps: true
scrape_interval: 60s
scrape_timeout: 5s
metrics_path: /metrics/stat
scheme: http
follow_redirects: true
enable_http2: true
static_configs:
- targets:
- mynodered1:1880
- job_name: cpu
honor_timestamps: true
scrape_interval: 60s
scrape_timeout: 5s
metrics_path: /metrics/cpu
scheme: http
follow_redirects: true
enable_http2: true
static_configs:
- targets:
- mynodered1:1880