nixos-config/nixos/machines/chungus/telemetry/opentelemetry.nix

{ pkgs, config, ... }:
{

  networking.firewall.interfaces.wg0.allowedTCPPorts = [ 4317 ];
  networking.firewall.interfaces.wg0.allowedUDPPorts = [ 4317 ];

  services.opentelemetry-collector = {
    enable = true;
    package = pkgs.unstable.opentelemetry-collector-contrib;
    settings = {
      receivers = {

        # receive metrics from other open telemetry collectors
        otlp.protocols.grpc.endpoint = "0.0.0.0:4317";

        # provide a influxdb sink
        influxdb = {
          endpoint = "127.0.0.1:8088";
        };

        # scrape opentelemetry-colectors metrics
        prometheus.config.scrape_configs = [
          {
            job_name = "netdata";
            scrape_interval = "10s";
            metrics_path = "/api/v1/allmetrics";
            params.format = [ "prometheus" ];
            static_configs = [{
              targets = [ "127.0.0.1:19999" ];
              labels = {
                service = "netdata";
                server = config.networking.hostName;
              };
            }];
          }
          {
            job_name = "otelcol";
            scrape_interval = "10s";
            static_configs = [{
              targets = [ "127.0.0.1:8100" ];
              labels = {
                service = "otelcol";
                server = config.networking.hostName;
              };
            }];
            metric_relabel_configs = [
              {
                source_labels = [ "__name__" ];
                regex = ".*grpc_io.*";
                action = "drop";
              }
            ];
          }
          {
            job_name = "node";
            static_configs = [{
              targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
              labels = {
                # todo : this is not really needed (right?)
                service = "node-exporter";
                # todo : use a processor for this
                server = config.networking.hostName;
              };
            }];
          }
          {
            # see https://www.home-assistant.io/integrations/prometheus/
            job_name = "home-assistant";
            scrape_interval = "60s";
            metrics_path = "/api/prometheus";
            bearer_token_file = toString config.sops.secrets.hass_long_term_token.path;
            static_configs = [{
              targets = [ "localhost:8123" ];
              labels = {
                service = "hass";
                server = config.networking.hostName;
              };
            }];
          }
        ];
      };

      exporters = {
        # provide prometheus sink under `/metrics` to 
        prometheus = {
          endpoint = "127.0.0.1:8090";
        };
      };

      service = {
        pipelines.metrics = {
          #receivers = [ "otlp" "influxdb" "prometheus" ];
          receivers = [ "otlp" "influxdb" ];
          exporters = [ "prometheus" ];
        };
        # open telemetries own metrics?
        telemetry.metrics.address = "0.0.0.0:8100";
      };
    };
  };
}