build01: add telegraf profile

This commit is contained in:
Jörg Thalheim 2021-01-18 22:20:07 +01:00
parent edd87968c2
commit 3c931630ad
No known key found for this signature in database
GPG key ID: 003F2096411B5F92
3 changed files with 150 additions and 1 deletions

View file

@ -26,7 +26,7 @@ in
../profiles/common.nix
../profiles/docker.nix
../profiles/telegraf
../services/hound
] ++ userImports;

View file

@ -0,0 +1,83 @@
{ pkgs, lib, config, ... }: let
isVM = lib.any (mod: mod == "xen-blkfront" || mod == "virtio_console") config.boot.initrd.kernelModules;
in {
config = {
systemd.services.telegraf.path = [ pkgs.nvme-cli ];
networking.firewall.allowedTCPPorts = [ 9273 ];
services.telegraf = {
enable = true;
extraConfig = {
agent.interval = "60s";
inputs = {
kernel_vmstat = {};
smart = lib.mkIf (!isVM) {
path = pkgs.writeShellScript "smartctl" ''
exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
'';
};
system = {};
mem = {};
file = [{
data_format = "influx";
file_tag = "name";
files = [ "/var/log/telegraf/*" ];
}] ++ lib.optional (lib.any (fs: fs == "ext4") config.boot.supportedFilesystems) {
name_override = "ext4_errors";
files = [ "/sys/fs/ext4/*/errors_count" ];
data_format = "value";
};
exec = lib.optionalAttrs (lib.any (fs: fs == "zfs") config.boot.supportedFilesystems) {
## Commands array
commands = [
(pkgs.writeScript "zpool-health" ''
#!${pkgs.gawk}/bin/awk -f
BEGIN {
while ("${pkgs.zfs}/bin/zpool status" | getline) {
if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
if ($1 ~ /errors:/) {
if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
}
}
}
'')
(pkgs.runCommandNoCC "mdraid-health" {
buildInputs = [ pkgs.bash ];
} ''
install -m755 ${./mdraid-health.sh} $out
patchShebangs $out
'')
];
data_format = "influx";
};
systemd_units = {};
swap = {};
disk.tagdrop = {
fstype = [ "tmpfs" "ramfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs" ];
device = [ "rpc_pipefs" "lxcfs" "nsfs" "borgfs" ];
};
};
outputs.prometheus_client = {
listen = ":9273";
metric_version = 2;
};
};
};
security.sudo.extraRules = lib.mkIf (!isVM) [{
users = [ "telegraf" ];
commands = [ {
command = "${pkgs.smartmontools}/bin/smartctl";
options = [ "NOPASSWD" ];
}];
}];
# avoid logging sudo use
security.sudo.configFile = ''
Defaults:telegraf !syslog,!pam_session
'';
# create dummy file to avoid telegraf errors
systemd.tmpfiles.rules = [
"f /var/log/telegraf/dummy 0444 root root - -"
];
};
}

View file

@ -0,0 +1,66 @@
#!/usr/bin/env bash
#
# telegraf plugin for mdstat monitoring
#
# output fields:
# - measurement: mdstat
# - tags:
# - host: hostname
# - dev: md device name (md0, md1 etc)
# - fields:
# - mismatch_cnt: number of mismatched sectors during latest check
# - state: clear / inactive / suspended / readonly / read-auto / clean / active / write-pending / active-idle
# - active_disks: number of active disks in array
# - degraded_disks: number of faulty disks in array (0 = healthy)
# - total_disks: number of disks in array
#
# see https://www.kernel.org/doc/html/v4.15/admin-guide/md.html for more
#
# state values:
# - clear: no devices, no size, no level
# - inactive: may have some settings, but array is not active all IO results in error
# - suspended: all IO requests will block. the array can be reconfigured.
# - readonly: no resync can happen. no superblocks get written. Write requests fail
# - read-auto: like readonly, but behaves like clean on a write request.
# - clean: no pending writes, but otherwise active.
# - active: fully active: IO and resync can be happening. when written to inactive array, starts with resync
# - write-pending: clean, but writes are blocked waiting for active to be written.
# - active-idle: like active, but no writes have been seen for a while
#
#
# sample /proc/mdstat
#
# Personalities : [raid1] [linear] [multipath] [raid0] [raid6] [raid5] [raid4] [raid10]
# md2 : active raid1 sdb3[1] sda3[0]
# 1936079936 blocks super 1.2 [2/2] [UU]
# bitmap: 2/15 pages [8KB], 65536KB chunk
#
# md1 : active raid1 sdb2[1] sda2[0]
# 523712 blocks super 1.2 [2/2] [UU]
#
# md0 : active raid1 sdb1[1] sda1[0]
# 16760832 blocks super 1.2 [2/2] [UU]
#
# sample /sys/block/md0/uevent
#
# MAJOR=9
# MINOR=0
# DEVNAME=md0
# DEVTYPE=disk
HOST=$(< /proc/sys/kernel/hostname)
for MD_SYS_FOLDER in /sys/block/md*; do
eval $(< "${MD_SYS_FOLDER}/uevent")
MD_DEV=${DEVNAME}
[ -z "${MD_DEV}" ] && continue
MISMATCH_CNT=$(< "${MD_SYS_FOLDER}/md/mismatch_cnt")
STATE=$(< "${MD_SYS_FOLDER}/md/array_state")
DEGRADED_DISKS=$(< "${MD_SYS_FOLDER}/md/degraded")
TOTAL_DISKS=$(< "${MD_SYS_FOLDER}/md/raid_disks")
ACTIVE_DISKS=$(expr $TOTAL_DISKS - $DEGRADED_DISKS)
echo "mdstat,host=${HOST},dev=${MD_DEV} mismatch_cnt=${MISMATCH_CNT}i,state=\"${STATE}\",active_disks=${ACTIVE_DISKS}i,degraded_disks=${DEGRADED_DISKS}i,total_disks=${TOTAL_DISKS}i"
done