nixos-config/nixos/machines/chungus/telemetry/telegraf.nix

{ config, pkgs, ... }:
let
  urls = [
    { url = "https://bitwarden.ingolf-wagner.de"; path = ""; }
    { url = "https://flix.ingolf-wagner.de"; path = "web/index.html"; }
    { url = "https://git.ingolf-wagner.de"; path = ""; }
    { url = "https://ingolf-wagner.de"; path = ""; }
    { url = "https://nextcloud.ingolf-wagner.de"; path = "login"; }
    { url = "https://tech.ingolf-wagner.de"; path = ""; }
    { url = "https://matrix.ingolf-wagner.de"; path = ""; }
  ];

in
{
  systemd.services.telegraf.path = [ pkgs.inetutils ];

  services.telegraf = {
    enable = true;
    extraConfig = {
      #outputs.prometheus_client = {
      #  listen = ":9273";
      #  metric_version = 2;
      #};
      outputs.influxdb_v2 = {
        urls = [ "http://127.0.0.1:8088" ];
      };

      global_tags = {
        service = "telegraf";
        server = config.networking.hostName;
      };

      # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins
      inputs = {
        cpu = { };
        diskio = { };
        smart.attributes = true;
        x509_cert = [{
          sources = (map (url: "${url.url}:443") urls);
          interval = "30m"; # agent.interval = "10s" is default
        }];
        http_response =
          let fullUrls = map ({ url, path }: "${url}/${path}") urls;
          in [{ urls = fullUrls; }];
        processes = { };
        system = { };
        systemd_units = { };
        internet_speed.interval = "10m";
        nginx.urls = [ "http://localhost/nginx_status" ];
        ping = [{ urls = [ "10.100.0.1" ]; }]; # actually important to make pepe visible over wireguard
      };
    };
  };

  # todo : do this prometheus
  services.prometheus.ruleFiles = [
    (pkgs.writeText "telegraf.yml" (builtins.toJSON {
      groups = [
        {
          name = "telegraf";
          rules = [
            {
              alert = "HttpResponseNotOk";
              expr = "0 * (http_response_http_response_code != 200) + 1";
              for = "5m";
              labels.severity = "page";
              annotations = {
                summary = "{{ $labels.exported_server }} does not return Ok";
                description = "{{ $labels.exported_server }} does not return Ok for more than 5 minutes";
              };
            }
            {
              alert = "CertificatExpires";
              expr = ''x509_cert_expiry{issuer_common_name="R3"} < ${toString (60 * 60 * 24 * 5)}'';
              for = "1d";
              labels.severity = "page";
              annotations = {
                summary = "{{ $labels.san }} does Expire Soon";
                description = "{{ $labels.san }} does expire in less than 5 days";
              };
            }
          ];
        }
      ];
    }))
  ];




}