# NixOps configuration for the hosts running Prometheus on a Cardano node { config, pkgs, lib, ... }: { services = { prometheus = { enable = true; webExternalUrl = "https://monitoring.mcwhirter.io/prometheus/"; extraFlags = ["--storage.tsdb.retention.time 8760h"]; exporters = { node = { enable = true; openFirewall = true; enabledCollectors = [ "systemd" "tcpstat" "conntrack" "diskstats" "entropy" "filefd" "filesystem" "loadavg" "meminfo" "netdev" "netstat" "stat" "time" "ntp" "timex" "vmstat" "logind" "interrupts" "ksmd" "processes" ]; }; }; #alertmanager = { # enable = true; # webExternalUrl = "https://monitoring.mcwhirter.io/alertmanager/"; # configuration = [ # ]; #}; #alertmanagers = [ { # scheme = "http"; # path_prefix = "/"; # static_configs = [ { # targets = [ "airgead.mcwhirter.io:9093" ]; # } ]; #} ]; rules = [ (builtins.toJSON { groups = [ { name = "system"; rules = [ { alert = "node_down"; expr = "up == 0"; for = "5m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Node is down."; description = "{{$labels.alias}} has been down for more than 5 minutes."; }; } { alert = "node_systemd_service_failed"; expr = ''node_systemd_unit_state{state="failed"} == 1''; for = "4m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start."; description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}."; }; } { alert = "node_filesystem_full_90percent"; expr = '' sort(node_filesystem_free_bytes{device!="ramfs"} < node_filesystem_size_bytes{device!="ramfs"} * 0.1) / 1024^3''; for = "5m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Filesystem is running out of space soon."; description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem."; }; } { alert = "node_filesystem_full_in_4h"; expr = '' predict_linear(node_filesystem_free_bytes{device!="ramfs",device!="tmpfs",fstype!="autofs",fstype!="cd9660"}[4h], 4*3600) <= 0''; for = "5m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours."; description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours"; }; } { alert = "node_filedescriptors_full_in_3h"; expr = "predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum"; for = "20m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours."; description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours"; }; } { alert = "node_load1_90percent"; expr = '' node_load1 / on(alias) count(node_cpu_seconds_total{mode="system"}) by (alias) >= 0.9''; for = "1h"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Running on high load."; description = "{{$labels.alias}} is running with > 90% total load for at least 1h."; }; } { alert = "node_cpu_util_90percent"; expr = '' 100 - (avg by (alias) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90''; for = "1h"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: High CPU utilization."; description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h."; }; } { alert = "node_ram_using_99percent"; expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.01"; for = "30m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Using lots of RAM."; description = "{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now."; }; } { alert = "node_swap_using_80percent"; expr = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.8"; for = "10m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Running out of swap soon."; description = "{{$labels.alias}} is using 80% of its swap space for at least 10 minutes now."; }; } { alert = "node_time_unsync"; expr = "abs(node_timex_offset_seconds) > 0.050 or node_timex_sync_status != 1"; for = "1m"; labels = {severity = "page";}; annotations = { summary = "{{$labels.alias}}: Clock out of sync with NTP"; description = "{{$labels.alias}} Local clock offset is too large or out of sync with NTP"; }; } ]; } ]; }) ]; scrapeConfigs = [ { job_name = "prometheus"; scrape_interval = "5s"; static_configs = [ { targets = ["localhost:9090"]; labels = {alias = "prometheus";}; } ]; } { job_name = "cardano-node"; scrape_interval = "10s"; static_configs = [ { targets = ["127.0.0.1:12798"]; labels = {alias = "airgead";}; } ]; } { job_name = "node"; scrape_interval = "10s"; static_configs = [ { targets = ["airgead.mcwhirter.io:9100"]; labels = {alias = "airgead.mcwhirter.io";}; } ]; } ]; }; }; }