{ config, pkgs, lib, ... }: {

  services.nginx = {
    enable = true;
    statusPage = true;
    virtualHosts = {
      "prometheus.workhorse.private" = {
        locations."/" = { proxyPass = "http://workhorse.private:9090"; };
      };
    };
  };

  services.prometheus = {
    enable = true;
    # keep data for 30 days
    extraFlags = [ "--storage.tsdb.retention.time=30d" ];
    scrapeConfigs = [
      {
        job_name = "nginx";
        scrape_interval = "8s";
        static_configs = [
          {
            targets = [ "sputnik.private:9113" ];
            labels = {
              service = "nginx";
              server = "sputnik";
            };
          }
          {
            targets = [ "sputnik.private:9113" ];
            labels = {
              service = "nginx";
              server = "sputnik";
            };
          }
          {
            targets = [ "workhorse.private:9113" ];
            labels = {
              service = "nginx";
              server = "sputnik";
            };
          }
        ];
      }
      {
        job_name = "netdata";
        metrics_path = "/api/v1/allmetrics";
        params.format = [ "prometheus" ];
        scrape_interval = "5s";
        static_configs = [
          {
            targets = [ "pepe.private:19999" ];
            labels = {
              service = "netdata";
              server = "pepe";
            };
          }
          {
            targets = [ "sputnik.private:19999" ];
            labels = {
              service = "netdata";
              server = "sputnik";
            };
          }
          {
            targets = [ "workhorse.private:19999" ];
            labels = {
              service = "netdata";
              server = "workhorse";
            };
          }
        ];
      }
      {
        job_name = "gogs";
        metrics_path = "/-/metrics";
        params.format = [ "prometheus" ];
        scrape_interval = "10s";
        static_configs = [{
          targets = [ "workhorse.private:3000" ];
          labels = {
            service = "gogs";
            server = "kruck";
          };
        }];
      }
      {
        job_name = "home-assistant";
        scrape_interval = "60s";
        metrics_path = "/api/prometheus";
        # you can create this token on your user profile page
        # http://pepe.private:8123/profile
        bearer_token =
          lib.fileContents <secrets/prometheus/home-assistant/api_token>;
        static_configs = [{
          targets = [ "pepe.private:8123" ];
          labels = {
            service = "hass";
            server = "pepe";
            city = "essen";
          };
        }];
      }
    ];
  };
}