modules/nixos/monitoring: add ofborg prometheus and eval queue alert

modules/nixos/monitoring/alert-rules.nix

@@ -26,6 +26,12 @@
           annotations.description = "{{$labels.host}} should have a running {{$labels.name}}";
         };
 
+        OfBorgEvalQueue = {
+          expr = ''ofborg_queue_evaluator_waiting > (2 * ofborg_queue_evaluator_consumers)'';
+          for = "1h";
+          annotations.description = "ofborg evaluator queue is more than 2x the number of evaluators";
+        };
+
         SmartErrors.expr = lib.mkForce ''smart_device_health_ok{enabled!="Disabled", host!="build01"} != 1'';
       };
   };

modules/nixos/monitoring/telegraf.nix

@@ -41,5 +41,8 @@
           timeout = "10s";
         })
         hosts;
+    prometheus.urls = [
+      "https://events.ofborg.org/prometheus.php"
+    ];
   };
 }