nixos-config/nixos/machines/pepe/telegraf.nix

{ pkgs, ... }:
let
  urls = [
    { url = "https://bitwarden.ingolf-wagner.de"; path = ""; }
    { url = "https://flix.ingolf-wagner.de"; path = "web/index.html"; }
    { url = "https://git.ingolf-wagner.de"; path = ""; }
    { url = "https://ingolf-wagner.de"; path = ""; }
    { url = "https://nextcloud.ingolf-wagner.de"; path = "login"; }
    { url = "https://tech.ingolf-wagner.de"; path = ""; }
  ];

in
{
  services.telegraf = {
    enable = true;
    extraConfig = {
      outputs.prometheus_client = {
        listen = ":9273";
        metric_version = 2;
      };
      # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins
      inputs = {
        cpu = { };
        diskio = { };
        x509_cert = [{
          sources = (map (url: "${url.url}:443") urls);
          interval = "30m"; # agent.interval = "10s" is default
        }];
        http_response =
          let fullUrls = map ({ url, path }: "${url}/${path}") urls;
          in [{ urls = fullUrls; }];
        processes = { };
        systemd_units = { };
        internet_speed.interval = "50m";
        nginx.urls = [ "http://localhost/nginx_status" ];
        ping = [{ urls = [ "10.100.0.1" ]; }]; # actually important to make pepe visible over wireguard
      };
    };
  };

  services.prometheus.scrapeConfigs = [
    {
      # see https://www.home-assistant.io/integrations/prometheus/
      job_name = "telgraf";
      metrics_path = "/metrics";
      static_configs = [{
        targets = [ "localhost:9273" ];
        labels = {
          service = "telegraf";
          server = "pepe";
        };
      }];
    }
  ];

  services.prometheus.ruleFiles = [
    (pkgs.writeText "telegraf.yml" (builtins.toJSON {
      groups = [
        {
          name = "telegraf";
          rules = [
            {
              alert = "HttpResponseNotOk";
              expr = "0 * (http_response_http_response_code != 200) + 1";
              for = "5m";
              labels.severity = "page";
              annotations = {
                summary = "{{ $labels.exported_server }} does not return Ok";
                description = "{{ $labels.exported_server }} does not return Ok for more than 5 minutes";
              };
            }
            {
              alert = "CertificatExpires";
              expr = ''x509_cert_expiry{issuer_common_name="R3"} < ${toString (60 * 60 * 24 * 5)}'';
              for = "1d";
              labels.severity = "page";
              annotations = {
                summary = "{{ $labels.san }} does Expire Soon";
                description = "{{ $labels.san }} does expire in less than 5 days";
              };
            }
          ];
        }
      ];
    }))
  ];




}