{ pkgs, ... }: let urls = [ { url = "https://bitwarden.ingolf-wagner.de"; path = ""; } { url = "https://flix.ingolf-wagner.de"; path = "web/index.html"; } { url = "https://git.ingolf-wagner.de"; path = ""; } { url = "https://ingolf-wagner.de"; path = ""; } { url = "https://nextcloud.ingolf-wagner.de"; path = "login"; } { url = "https://tech.ingolf-wagner.de"; path = ""; } ]; in { services.telegraf = { enable = true; extraConfig = { outputs.prometheus_client = { listen = ":9273"; metric_version = 2; }; # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins inputs = { x509_cert = [{ sources = (map (url: "${url.url}:443") urls); interval = "30m"; # agent.interval = "10s" is default }]; http_response = let fullUrls = map ({ url, path }: "${url}/${path}") urls; in [{ urls = fullUrls; }]; processes = { }; systemd_units = { }; internet_speed.interval = "50m"; nginx.urls = [ "http://localhost/nginx_status" ]; ping = [{ urls = [ "10.100.0.1" ]; }]; # actually important to make pepe visible over wireguard }; }; }; services.prometheus.scrapeConfigs = [ { # see https://www.home-assistant.io/integrations/prometheus/ job_name = "telgraf"; metrics_path = "/metrics"; static_configs = [{ targets = [ "localhost:9273" ]; labels = { service = "telegraf"; server = "pepe"; }; }]; } ]; services.prometheus.ruleFiles = [ (pkgs.writeText "telegraf.yml" (builtins.toJSON { groups = [ { name = "telegraf"; rules = [ { alert = "HttpResponseNotOk"; expr = "0 * (http_response_http_response_code != 200) + 1"; for = "5m"; labels.severity = "page"; annotations = { summary = "{{ $labels.exported_server }} does not return Ok"; description = "{{ $labels.exported_server }} does not return Ok for more than 5 minutes"; }; } { alert = "CertificatExpires"; expr = ''x509_cert_expiry{issuer_common_name="R3"} < ${toString (60 * 60 * 24 * 5)}''; for = "1d"; labels.severity = "page"; annotations = { summary = "{{ $labels.san }} does Expire Soon"; description = "{{ $labels.san }} does expire in less than 5 days"; }; } ]; } ]; })) ]; }