nixos-config/components/monitor/opentelemetry.nix

{
  pkgs,
  config,
  lib,
  ...
}:
with lib;
with types;
let
  cfg = config.components.monitor.opentelemetry;
in
{
  options.components.monitor.opentelemetry = {
    enable = mkOption {
      type = bool;
      default = config.components.monitor.enable;
      description = "weather or not to use opentelemetry";
    };
    receiver.endpoint = mkOption {
      type = nullOr str;
      default = null;
      description = "endpoint to receive the opentelementry data from other collectors";
    };
    exporter.endpoint = mkOption {
      type = nullOr str;
      default = null;
      description = "endpoint to ship opentelementry data too";
    };
    exporter.debug = mkOption {
      type = nullOr (enum [
        "logs"
        "metrics"
      ]);
      default = null;
      description = "enable debug exporter.";
    };
    metrics.endpoint = mkOption {
      type = str;
      default = "127.0.0.1:8100";
      description = "endpoint on where to provide opentelementry metrics";
    };
  };

  config = mkMerge [

    (mkIf config.components.monitor.opentelemetry.enable {
      services.opentelemetry-collector = {
        enable = true;
        package = pkgs.opentelemetry-collector-contrib;
      };
    })

    # add default tags to metrics
    # todo : make sure we filter out metrics from otlp receivers
    (mkIf config.components.monitor.enable {
      services.opentelemetry-collector.settings = {

        processors = {

          # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md
          "resourcedetection/system" = {
            detectors = [ "system" ];
            override = false;
            system.hostname_sources = [ "os" ];
          };

          metricstransform.transforms = [
            {
              include = ".*";
              match_type = "regexp";
              action = "update";
              operations = [
                {
                  action = "add_label";
                  new_label = "machine";
                  new_value = config.networking.hostName;
                }
              ];
            }
          ];
        };
      };
    })
    (mkIf config.components.monitor.metrics.enable {
      services.opentelemetry-collector.settings = {
        service.pipelines.metrics.processors = [
          "metricstransform"
          "resourcedetection/system"
        ];
      };
    })
    (mkIf config.components.monitor.logs.enable {
      services.opentelemetry-collector.settings = {
        service.pipelines.logs.processors = [ "resourcedetection/system" ];
      };
    })

    (mkIf (config.components.monitor.opentelemetry.exporter.debug != null) {
      services.opentelemetry-collector.settings = {
        exporters.debug = {
          verbosity = "detailed";
          sampling_initial = 5;
          sampling_thereafter = 200;
        };
        service.pipelines.${config.components.monitor.opentelemetry.exporter.debug} = {
          exporters = [ "debug" ];
        };

      };
    })

    # ship to next instance
    (mkIf (config.components.monitor.opentelemetry.exporter.endpoint != null) {
      services.opentelemetry-collector.settings = {
        exporters.otlp = {
          endpoint = cfg.exporter.endpoint;
          tls.insecure = true;
        };
      };
    })
    (mkIf
      (
        config.components.monitor.opentelemetry.exporter.endpoint != null
        && config.components.monitor.logs.enable
      )
      {
        services.opentelemetry-collector.settings = {
          service.pipelines.logs.exporters = [ "otlp" ];
        };
      }
    )
    (mkIf
      (
        config.components.monitor.opentelemetry.exporter.endpoint != null
        && config.components.monitor.metrics.enable
      )
      {
        services.opentelemetry-collector.settings = {
          service.pipelines.metrics.exporters = [ "otlp" ];
        };
      }
    )

    # ship from other instance
    (mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) {
      services.opentelemetry-collector.settings = {
        receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint;
      };
    })
    (mkIf
      (
        config.components.monitor.opentelemetry.receiver.endpoint != null
        && config.components.monitor.logs.enable
      )
      {
        services.opentelemetry-collector.settings = {
          service.pipelines.logs.receivers = [ "otlp" ];
        };
      }
    )
    (mkIf
      (
        config.components.monitor.opentelemetry.receiver.endpoint != null
        && config.components.monitor.metrics.enable
      )
      {
        services.opentelemetry-collector.settings = {
          service.pipelines.metrics.receivers = [ "otlp" ];
        };
      }
    )

    # scrape opentelemetry-colectors metrics
    # todo: this should be collected another way (opentelemetry internal?)
    # todo : enable me only when metrics.endpoint is set.
    (mkIf config.components.monitor.metrics.enable {
      services.opentelemetry-collector.settings = {
        receivers = {
          prometheus.config.scrape_configs = [
            {
              job_name = "otelcol";
              scrape_interval = "10s";
              static_configs = [
                {
                  targets = [ cfg.metrics.endpoint ];
                }
              ];
              metric_relabel_configs = [
                {
                  source_labels = [ "__name__" ];
                  regex = ".*grpc_io.*";
                  action = "drop";
                }
              ];
            }
          ];
        };

        service = {
          pipelines.metrics = {
            receivers = [ "prometheus" ];
          };

          # todo : this should be automatically be collected
          # open telemetries own metrics?
          telemetry.metrics.address = cfg.metrics.endpoint;
        };

      };
    })
    (mkIf (!config.components.monitor.metrics.enable) {
      services.opentelemetry-collector.settings = {
        service.telemetry.metrics.level = "none";
      };
    })
  ];

}