nixos-config/nixos/machines/chungus/telemetry/prometheus.nix

{ config, pkgs, lib, ... }: {

  sops.secrets.hass_long_term_token.owner = "prometheus";

  services.nginx = {
    enable = true;
    statusPage = true;
    virtualHosts = {
      "prometheus.${config.networking.hostName}.private" = {
        extraConfig = ''
          allow ${config.tinc.private.subnet};
          deny all;
        '';
        locations."/" = { proxyPass = "http://localhost:${toString config.services.prometheus.port}"; };
      };
    };
  };

  services.prometheus = {
    checkConfig = "syntax-only";
    enable = true;
    # keep data for 30 days
    extraFlags = [ "--storage.tsdb.retention.time=90d" ];

    ruleFiles = [
      (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
        groups = [
          {
            name = "core";
            rules = [
              {
                alert = "InstanceDown";
                expr = "up == 0";
                for = "5m";
                labels.severity = "page";
                annotations = {
                  summary = "Instance {{ $labels.instance }} down";
                  description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.";
                };
              }
            ];
          }
          # todo : move this to open telemetry
          {
            name = "home-assistant";
            rules = [
              {
                record = "home_open_window_sum";
                expr = ''sum( homeassistant_binary_sensor_state{entity=~"binary_sensor\\.window_02_contact|binary_sensor\\.window_03_contact|binary_sensor\\.window_04_contact|binary_sensor\\.window_05_contact|binary_sensor\\.window_06_contact|binary_sensor\\.window_07_contact"} )'';
              }
            ] ++ (map
              (number:
                {
                  record = "home_at_least_n_windows_open";
                  expr = ''home_open_window_sum >= bool ${toString number}'';
                  labels.n = number;
                }) [ 1 2 3 ]);
          }
        ];
      }))
    ];

    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
        port = 9002;
      };
    };

    scrapeConfigs = [
      {
        job_name = "opentelemetry";
        metrics_path = "/metrics";
        scrape_interval = "10s";
        static_configs = [{ targets = [ "localhost:8090" ]; }];
      }
      #{
      #  job_name = "netdata";
      #  metrics_path = "/api/v1/allmetrics";
      #  params.format = [ "prometheus" ];
      #  scrape_interval = "5s";
      #  static_configs = [
      #    {
      #      targets = [ "localhost:19999" ];
      #      labels = {
      #        service = "netdata";
      #        server = config.networking.hostName;
      #      };
      #    }
      #  ];
      #}
      #{
      #  job_name = "node";
      #  static_configs = [{
      #    targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
      #    labels = {
      #      service = "node-exporter";
      #      server = config.networking.hostName;
      #    };
      #  }];
      #}
      #{
      #  # see https://www.home-assistant.io/integrations/prometheus/
      #  job_name = "home-assistant";
      #  scrape_interval = "60s";
      #  metrics_path = "/api/prometheus";
      #  bearer_token_file = toString config.sops.secrets.hass_long_term_token.path;
      #  static_configs = [{
      #    targets = [ "localhost:8123" ];
      #    labels = {
      #      service = "hass";
      #      server = config.networking.hostName;
      #    };
      #  }];
      #}
    ];
  };
}