nixos-config/nixos/components/monitor/opentelemetry.nix

{ pkgs, config, lib, ... }:
with lib;
with types;
let
  cfg = config.components.monitor.opentelemetry;
in
{
  options.components.monitor.opentelemetry = {
    receiver.endpoint = mkOption {
      type = nullOr str;
      default = null;
      description = "endpoint to receive the opentelementry data from other collectors";
    };
    exporter.endpoint = mkOption {
      type = nullOr str;
      default = null;
      description = "endpoint to ship opentelementry data too";
    };
    metrics.endpoint = mkOption {
      type = str;
      default = "127.0.0.1:8100";
      description = "endpoint on where to provide opentelementry metrics";
    };
  };

  config = mkMerge [
    (mkIf config.components.monitor.enable {
      services.opentelemetry-collector = {
        enable = true;
        package = pkgs.unstable.opentelemetry-collector-contrib;
      };
    })

    # add default tags to metrics
    # todo : make sure we filter out metrics from otlp receivers
    (mkIf config.components.monitor.enable {
      services.opentelemetry-collector.settings = {

        service.pipelines.metrics.processors = [ "metricstransform" "resourcedetection/system" ];
        service.pipelines.logs.processors = [ "resourcedetection/system" ];

        processors = {

          # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md
          "resourcedetection/system" = {
            detectors = [ "system" ];
            override = false;
            system.hostname_sources = [ "os" ];
          };

          metricstransform.transforms = [
            {
              include = ".*";
              match_type = "regexp";
              action = "update";
              operations = [{
                action = "add_label";
                new_label = "machine";
                new_value = config.networking.hostName;
              }];
            }
          ];
        };
      };
    })

    # ship to next instance
    (mkIf (config.components.monitor.opentelemetry.exporter.endpoint != null) {
      services.opentelemetry-collector.settings = {
        exporters.otlp = {
          endpoint = cfg.exporter.endpoint;
          tls.insecure = true;
        };
        service = {
          pipelines.metrics = {
            exporters = [ "otlp" ];
          };
          pipelines.logs = {
            exporters = [ "otlp" ];
          };
        };
      };
    })

    # ship from other instance
    (mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) {
      services.opentelemetry-collector.settings = {
        receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint;
        service = {
          pipelines.metrics = {
            receivers = [ "otlp" ];
          };
          pipelines.logs = {
            receivers = [ "otlp" ];
          };
        };
      };
    })

    # scrape opentelemetry-colectors metrics
    # todo: this should be collected another way (opentelemetry internal?)
    (mkIf config.components.monitor.enable {
      services.opentelemetry-collector.settings = {
        receivers = {
          prometheus.config.scrape_configs = [
            {
              job_name = "otelcol";
              scrape_interval = "10s";
              static_configs = [{
                targets = [ cfg.metrics.endpoint ];
              }];
              metric_relabel_configs = [
                {
                  source_labels = [ "__name__" ];
                  regex = ".*grpc_io.*";
                  action = "drop";
                }
              ];
            }
          ];
        };

        service = {
          pipelines.metrics = {
            receivers = [ "prometheus" ];
          };

          # todo : this should be automatically be collected
          # open telemetries own metrics?
          telemetry.metrics.address = cfg.metrics.endpoint;
        };

      };
    })
  ];

}