diff --git a/hosts/airgead.nix b/hosts/airgead.nix index 7a9c966..6fb092c 100644 --- a/hosts/airgead.nix +++ b/hosts/airgead.nix @@ -8,6 +8,7 @@ [ ../clusters/linode-encrypted.nix ../roles/cardano-node.nix + ../roles/prometheus_cardano-node.nix ]; deployment.targetHost = "172.105.184.221"; diff --git a/roles/cardano-node.nix b/roles/cardano-node.nix index f4defc6..a9674fd 100644 --- a/roles/cardano-node.nix +++ b/roles/cardano-node.nix @@ -13,6 +13,8 @@ in imports = [ ../secrets/cardano.nix + ../secrets/cardano/producers.nix + ./prometheus_cardano-node.nix "${sources.cardano-node}/nix/nixos" ]; @@ -25,15 +27,6 @@ in enable = true; environment = "ff"; hostAddr = "0.0.0.0"; - topology = builtins.toFile "topology.json" (builtins.toJSON { - Producers = [ - { - addr = "172.105.184.221"; - port = 3001; - valency = 1; - } - ]; - }); nodeConfig = config.services.cardano-node.environments.alpha1.nodeConfig // { hasPrometheus = [ "127.0.0.1" 12798 ]; setupScribes = [{ diff --git a/roles/prometheus_cardano-node.nix b/roles/prometheus_cardano-node.nix new file mode 100644 index 0000000..0ca2ac1 --- /dev/null +++ b/roles/prometheus_cardano-node.nix @@ -0,0 +1,241 @@ +# NixOps configuration for the hosts running Prometheus on a Cardano node + +{ config, pkgs, lib, ... }: + +{ + + imports = [ + ../secrets/cardano/grafana.nix + ]; + + services = { + prometheus = { + enable = true; + extraFlags = [ + "--storage.tsdb.retention.time 8760h" + ]; + alertmanagers = [ { + scheme = "http"; + path_prefix = "/"; + static_configs = [ { + targets = [ "airgead.mcwhirter.io:9093" ]; + } ]; + } ]; + rules = [ (builtins.toJSON { + groups = [ + { + name = "system"; + rules = [ + { + alert = "node_down"; + expr = "up == 0"; + for = "5m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Node is down."; + description = "{{$labels.alias}} has been down for more than 5 minutes."; + }; + } + { + alert = "node_systemd_service_failed"; + expr = "node_systemd_unit_state{state=\"failed\"} == 1"; + for = "4m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start."; + description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}."; + }; + } + { + alert = "node_filesystem_full_90percent"; + expr = "sort(node_filesystem_free_bytes{device!=\"ramfs\"} < node_filesystem_size_bytes{device!=\"ramfs\"} * 0.1) / 1024^3"; + for = "5m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Filesystem is running out of space soon."; + description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem."; + }; + } + { + alert = "node_filesystem_full_in_4h"; + expr = "predict_linear(node_filesystem_free_bytes{device!=\"ramfs\",device!=\"tmpfs\",fstype!=\"autofs\",fstype!=\"cd9660\"}[4h], 4*3600) <= 0"; + for = "5m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours."; + description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours"; + }; + } + { + alert = "node_filedescriptors_full_in_3h"; + expr = "predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum"; + for = "20m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours."; + description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours"; + }; + } + { + alert = "node_load1_90percent"; + expr = "node_load1 / on(alias) count(node_cpu_seconds_total{mode=\"system\"}) by (alias) >= 0.9"; + for = "1h"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Running on high load."; + description = "{{$labels.alias}} is running with > 90% total load for at least 1h."; + }; + } + { + alert = "node_cpu_util_90percent"; + expr = "100 - (avg by (alias) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100) >= 90"; + for = "1h"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: High CPU utilization."; + description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h."; + }; + } + { + alert = "node_ram_using_99percent"; + expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.01"; + for = "30m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Using lots of RAM."; + description = "{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now."; + }; + } + { + alert = "node_swap_using_80percent"; + expr = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.8"; + for = "10m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Running out of swap soon."; + description = "{{$labels.alias}} is using 80% of its swap space for at least 10 minutes now."; + }; + } + { + alert = "node_time_unsync"; + expr = "abs(node_timex_offset_seconds) > 0.050 or node_timex_sync_status != 1"; + for = "1m"; + labels = { + severity = "page"; + }; + annotations = { + summary = "{{$labels.alias}}: Clock out of sync with NTP"; + description = "{{$labels.alias}} Local clock offset is too large or out of sync with NTP"; + }; + } + ]; + } + ]; + })]; + scrapeConfigs = [ + { + job_name = "prometheus"; + scrape_interval = "5s"; + static_configs = [ + { + targets = [ + "localhost:9090" + ]; + } + ]; + } + { + job_name = "cardano-node"; + scrape_interval = "10s"; + static_configs = [ + { + targets = [ "127.0.0.1:12798" ]; + labels = { alias = "airgead"; }; + } + ]; + } + { + job_name = "node"; + scrape_interval = "10s"; + static_configs = [ + { + targets = [ + "airgead.mcwhirter.io:9100" + ]; + labels = { + alias = "airgead.mcwhirter.io"; + }; + } + ]; + } + ]; + }; + + grafana = { + enable = true; + addr = "0.0.0.0"; + domain = "grafana.mcwhirter.io"; + rootUrl = "https://grafana.mcwhirter.io/"; + security = { + adminPasswordFile = "/run/keys/grafana-apass"; # Where to find the password + }; + auth = { + anonymous = { + enable = true; # Allow anonymous access + }; + }; + provision = { + enable = true; + datasources = [ + { + type = "prometheus"; + name = "prometheus"; + url = "http://localhost:9090/prometheus"; + } + ]; + }; + }; + + nginx = { + enable = true; # Enable Nginx + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + virtualHosts."grafana.mcwhirter.io" = { # Grafana hostname + enableACME = true; # Use ACME certs + forceSSL = true; # Force SSL + locations."/".proxyPass = "http://localhost:3000/"; # Proxy Grafana + }; + }; + + }; + + users.groups.keys.members = [ "grafana" ]; # Required due to NixOps issue #1204 + + security.acme = { + acceptTerms = true; + certs = { + "grafana.mcwhirter.io".email = "craige@mcwhirter.io"; + }; + }; + +}