{ pkgs, config, ... }:
{

  services.opentelemetry-collector = {
    enable = true;
    package = pkgs.unstable.opentelemetry-collector-contrib;
    settings = {
      receivers = {

        # provide a influxdb sink
        influxdb = {
          endpoint = "127.0.0.1:8088";
        };

        # scrape opentelemetry-colectors metrics
        prometheus.config.scrape_configs = [
          {
            job_name = "netdata";
            scrape_interval = "10s";
            metrics_path = "/api/v1/allmetrics";
            params.format = [ "prometheus" ];
            static_configs = [{
              targets = [ "127.0.0.1:19999" ];
              labels = {
                service = "netdata";
                server = config.networking.hostName;
              };
            }];
          }
          {
            job_name = "otelcol";
            scrape_interval = "10s";
            static_configs = [{
              targets = [ "127.0.0.1:8100" ];
              labels = {
                service = "otelcol";
                server = config.networking.hostName;
              };
            }];
            metric_relabel_configs = [
              {
                source_labels = [ "__name__" ];
                regex = ".*grpc_io.*";
                action = "drop";
              }
            ];
          }
          {
            job_name = "node";
            static_configs = [{
              targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
              labels = {
                service = "node-exporter";
                server = config.networking.hostName;
              };
            }];
          }
        ];
      };

      exporters = {
        # provide prometheus sink under `/metrics` to 
        prometheus = {
          endpoint = "127.0.0.1:8090";
        };
        otlp = {
          endpoint = "10.100.0.2:4317"; # chungus
          tls.insecure = true;
        };
      };

      service = {
        pipelines.metrics = {
          receivers = [ "influxdb" "prometheus" ];
          exporters = [ "prometheus" "otlp" ];
        };
        # open telemetries own metrics?
        telemetry.metrics.address = "0.0.0.0:8100";
      };
    };
  };
}