mio-ops/profiles/prometheus.nix

204 lines
7.6 KiB
Nix

# NixOps configuration for the hosts running Prometheus on a Cardano node
{
config,
pkgs,
lib,
...
}: {
services = {
prometheus = {
enable = true;
webExternalUrl = "https://monitoring.mcwhirter.io/prometheus/";
extraFlags = ["--storage.tsdb.retention.time 8760h"];
exporters = {
node = {
enable = true;
openFirewall = true;
enabledCollectors = [
"systemd"
"tcpstat"
"conntrack"
"diskstats"
"entropy"
"filefd"
"filesystem"
"loadavg"
"meminfo"
"netdev"
"netstat"
"stat"
"time"
"ntp"
"timex"
"vmstat"
"logind"
"interrupts"
"ksmd"
"processes"
];
};
};
#alertmanager = {
# enable = true;
# webExternalUrl = "https://monitoring.mcwhirter.io/alertmanager/";
# configuration = [
# ];
#};
#alertmanagers = [ {
# scheme = "http";
# path_prefix = "/";
# static_configs = [ {
# targets = [ "airgead.mcwhirter.io:9093" ];
# } ];
#} ];
rules = [
(builtins.toJSON {
groups = [
{
name = "system";
rules = [
{
alert = "node_down";
expr = "up == 0";
for = "5m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Node is down.";
description = "{{$labels.alias}} has been down for more than 5 minutes.";
};
}
{
alert = "node_systemd_service_failed";
expr = ''node_systemd_unit_state{state="failed"} == 1'';
for = "4m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.";
description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}.";
};
}
{
alert = "node_filesystem_full_90percent";
expr = ''
sort(node_filesystem_free_bytes{device!="ramfs"} < node_filesystem_size_bytes{device!="ramfs"} * 0.1) / 1024^3'';
for = "5m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Filesystem is running out of space soon.";
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem.";
};
}
{
alert = "node_filesystem_full_in_4h";
expr = ''
predict_linear(node_filesystem_free_bytes{device!="ramfs",device!="tmpfs",fstype!="autofs",fstype!="cd9660"}[4h], 4*3600) <= 0'';
for = "5m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours.";
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours";
};
}
{
alert = "node_filedescriptors_full_in_3h";
expr = "predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum";
for = "20m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.";
description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours";
};
}
{
alert = "node_load1_90percent";
expr = ''
node_load1 / on(alias) count(node_cpu_seconds_total{mode="system"}) by (alias) >= 0.9'';
for = "1h";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Running on high load.";
description = "{{$labels.alias}} is running with > 90% total load for at least 1h.";
};
}
{
alert = "node_cpu_util_90percent";
expr = ''
100 - (avg by (alias) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
for = "1h";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: High CPU utilization.";
description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h.";
};
}
{
alert = "node_ram_using_99percent";
expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.01";
for = "30m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Using lots of RAM.";
description = "{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now.";
};
}
{
alert = "node_swap_using_80percent";
expr = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.8";
for = "10m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Running out of swap soon.";
description = "{{$labels.alias}} is using 80% of its swap space for at least 10 minutes now.";
};
}
{
alert = "node_time_unsync";
expr = "abs(node_timex_offset_seconds) > 0.050 or node_timex_sync_status != 1";
for = "1m";
labels = {severity = "page";};
annotations = {
summary = "{{$labels.alias}}: Clock out of sync with NTP";
description = "{{$labels.alias}} Local clock offset is too large or out of sync with NTP";
};
}
];
}
];
})
];
scrapeConfigs = [
{
job_name = "prometheus";
scrape_interval = "5s";
static_configs = [
{
targets = ["localhost:9090"];
labels = {alias = "prometheus";};
}
];
}
{
job_name = "cardano-node";
scrape_interval = "10s";
static_configs = [
{
targets = ["127.0.0.1:12798"];
labels = {alias = "airgead";};
}
];
}
{
job_name = "node";
scrape_interval = "10s";
static_configs = [
{
targets = ["airgead.mcwhirter.io:9100"];
labels = {alias = "airgead.mcwhirter.io";};
}
];
}
];
};
};
}