sercanto/mio-ops
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Activity

Provisioned initial node

Browse source
This commit is contained in:
Craige McWhirter 2020-05-26 22:13:24 +10:00
parent fa2e2d9a75
commit 1e089ddace
Signed by: sercanto
GPG key ID: 7DBA9F5689EFB6AA
3 changed files with 244 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
hosts/airgead.nix
View file

@ -8,6 +8,7 @@
[
../clusters/linode-encrypted.nix
../roles/cardano-node.nix
../roles/prometheus_cardano-node.nix
];
deployment.targetHost = "172.105.184.221";

11
roles/cardano-node.nix
View file

@ -13,6 +13,8 @@ in
imports = [
../secrets/cardano.nix
../secrets/cardano/producers.nix
./prometheus_cardano-node.nix
"${sources.cardano-node}/nix/nixos"
];
@ -25,15 +27,6 @@ in
enable = true;
environment = "ff";
hostAddr = "0.0.0.0";
topology = builtins.toFile "topology.json" (builtins.toJSON {
Producers = [
{
addr = "172.105.184.221";
port = 3001;
valency = 1;
}
];
});
nodeConfig = config.services.cardano-node.environments.alpha1.nodeConfig // {
hasPrometheus = [ "127.0.0.1" 12798 ];
setupScribes = [{

241
roles/prometheus_cardano-node.nix Normal file
View file

@ -0,0 +1,241 @@
# NixOps configuration for the hosts running Prometheus on a Cardano node
{ config, pkgs, lib, ... }:
{
imports = [
../secrets/cardano/grafana.nix
];
services = {
prometheus = {
enable = true;
extraFlags = [
"--storage.tsdb.retention.time 8760h"
];
alertmanagers = [ {
scheme = "http";
path_prefix = "/";
static_configs = [ {
targets = [ "airgead.mcwhirter.io:9093" ];
} ];
} ];
rules = [ (builtins.toJSON {
groups = [
{
name = "system";
rules = [
{
alert = "node_down";
expr = "up == 0";
for = "5m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Node is down.";
description = "{{$labels.alias}} has been down for more than 5 minutes.";
};
}
{
alert = "node_systemd_service_failed";
expr = "node_systemd_unit_state{state=\"failed\"} == 1";
for = "4m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.";
description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}.";
};
}
{
alert = "node_filesystem_full_90percent";
expr = "sort(node_filesystem_free_bytes{device!=\"ramfs\"} < node_filesystem_size_bytes{device!=\"ramfs\"} * 0.1) / 1024^3";
for = "5m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Filesystem is running out of space soon.";
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem.";
};
}
{
alert = "node_filesystem_full_in_4h";
expr = "predict_linear(node_filesystem_free_bytes{device!=\"ramfs\",device!=\"tmpfs\",fstype!=\"autofs\",fstype!=\"cd9660\"}[4h], 4*3600) <= 0";
for = "5m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours.";
description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours";
};
}
{
alert = "node_filedescriptors_full_in_3h";
expr = "predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum";
for = "20m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.";
description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours";
};
}
{
alert = "node_load1_90percent";
expr = "node_load1 / on(alias) count(node_cpu_seconds_total{mode=\"system\"}) by (alias) >= 0.9";
for = "1h";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Running on high load.";
description = "{{$labels.alias}} is running with > 90% total load for at least 1h.";
};
}
{
alert = "node_cpu_util_90percent";
expr = "100 - (avg by (alias) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100) >= 90";
for = "1h";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: High CPU utilization.";
description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h.";
};
}
{
alert = "node_ram_using_99percent";
expr = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.01";
for = "30m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Using lots of RAM.";
description = "{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now.";
};
}
{
alert = "node_swap_using_80percent";
expr = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.8";
for = "10m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Running out of swap soon.";
description = "{{$labels.alias}} is using 80% of its swap space for at least 10 minutes now.";
};
}
{
alert = "node_time_unsync";
expr = "abs(node_timex_offset_seconds) > 0.050 or node_timex_sync_status != 1";
for = "1m";
labels = {
severity = "page";
};
annotations = {
summary = "{{$labels.alias}}: Clock out of sync with NTP";
description = "{{$labels.alias}} Local clock offset is too large or out of sync with NTP";
};
}
];
}
];
})];
scrapeConfigs = [
{
job_name = "prometheus";
scrape_interval = "5s";
static_configs = [
{
targets = [
"localhost:9090"
];
}
];
}
{
job_name = "cardano-node";
scrape_interval = "10s";
static_configs = [
{
targets = [ "127.0.0.1:12798" ];
labels = { alias = "airgead"; };
}
];
}
{
job_name = "node";
scrape_interval = "10s";
static_configs = [
{
targets = [
"airgead.mcwhirter.io:9100"
];
labels = {
alias = "airgead.mcwhirter.io";
};
}
];
}
];
};
grafana = {
enable = true;
addr = "0.0.0.0";
domain = "grafana.mcwhirter.io";
rootUrl = "https://grafana.mcwhirter.io/";
security = {
adminPasswordFile = "/run/keys/grafana-apass"; # Where to find the password
};
auth = {
anonymous = {
enable = true; # Allow anonymous access
};
};
provision = {
enable = true;
datasources = [
{
type = "prometheus";
name = "prometheus";
url = "http://localhost:9090/prometheus";
}
];
};
};
nginx = {
enable = true; # Enable Nginx
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
virtualHosts."grafana.mcwhirter.io" = { # Grafana hostname
enableACME = true; # Use ACME certs
forceSSL = true; # Force SSL
locations."/".proxyPass = "http://localhost:3000/"; # Proxy Grafana
};
};
};
users.groups.keys.members = [ "grafana" ]; # Required due to NixOps issue #1204
security.acme = {
acceptTerms = true;
certs = {
"grafana.mcwhirter.io".email = "craige@mcwhirter.io";
};
};
}