nixos-config/components/monitor/opentelemetry.nix
2024-06-07 22:56:52 +02:00

205 lines
5.8 KiB
Nix

{ pkgs, config, lib, ... }:
with lib;
with types;
let
cfg = config.components.monitor.opentelemetry;
in
{
options.components.monitor.opentelemetry = {
enable = mkOption {
type = bool;
default = config.components.monitor.enable;
description = "weather or not to use opentelemetry";
};
receiver.endpoint = mkOption {
type = nullOr str;
default = null;
description = "endpoint to receive the opentelementry data from other collectors";
};
exporter.endpoint = mkOption {
type = nullOr str;
default = null;
description = "endpoint to ship opentelementry data too";
};
exporter.debug = mkOption {
type = nullOr (enum [ "logs" "metrics" ]);
default = null;
description = "enable debug exporter.";
};
metrics.endpoint = mkOption {
type = str;
default = "127.0.0.1:8100";
description = "endpoint on where to provide opentelementry metrics";
};
};
config = mkMerge [
(mkIf config.components.monitor.opentelemetry.enable {
services.opentelemetry-collector = {
enable = true;
package = pkgs.opentelemetry-collector-contrib;
};
})
# add default tags to metrics
# todo : make sure we filter out metrics from otlp receivers
(mkIf config.components.monitor.enable {
services.opentelemetry-collector.settings = {
processors = {
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md
"resourcedetection/system" = {
detectors = [ "system" ];
override = false;
system.hostname_sources = [ "os" ];
};
metricstransform.transforms = [
{
include = ".*";
match_type = "regexp";
action = "update";
operations = [{
action = "add_label";
new_label = "machine";
new_value = config.networking.hostName;
}];
}
];
};
};
})
(mkIf config.components.monitor.metrics.enable {
services.opentelemetry-collector.settings = {
service.pipelines.metrics.processors = [
"metricstransform"
"resourcedetection/system"
];
};
})
(mkIf config.components.monitor.logs.enable {
services.opentelemetry-collector.settings = {
service.pipelines.logs.processors = [ "resourcedetection/system" ];
};
})
(mkIf (config.components.monitor.opentelemetry.exporter.debug != null) {
services.opentelemetry-collector.settings = {
exporters.debug = {
verbosity = "detailed";
sampling_initial = 5;
sampling_thereafter = 200;
};
service.pipelines.${config.components.monitor.opentelemetry.exporter.debug} = {
exporters = [ "debug" ];
};
};
})
# ship to next instance
(mkIf (config.components.monitor.opentelemetry.exporter.endpoint != null) {
services.opentelemetry-collector.settings = {
exporters.otlp = {
endpoint = cfg.exporter.endpoint;
tls.insecure = true;
};
};
})
(mkIf
(
config.components.monitor.opentelemetry.exporter.endpoint != null &&
config.components.monitor.logs.enable
)
{
services.opentelemetry-collector.settings = {
service.pipelines.logs.exporters = [ "otlp" ];
};
})
(mkIf
(
config.components.monitor.opentelemetry.exporter.endpoint != null &&
config.components.monitor.metrics.enable
)
{
services.opentelemetry-collector.settings = {
service.pipelines.metrics.exporters = [ "otlp" ];
};
})
# ship from other instance
(mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) {
services.opentelemetry-collector.settings = {
receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint;
};
})
(mkIf
(
config.components.monitor.opentelemetry.receiver.endpoint != null &&
config.components.monitor.logs.enable
)
{
services.opentelemetry-collector.settings = {
service.pipelines.logs.receivers = [ "otlp" ];
};
})
(mkIf
(
config.components.monitor.opentelemetry.receiver.endpoint != null &&
config.components.monitor.metrics.enable
)
{
services.opentelemetry-collector.settings = {
service.pipelines.metrics.receivers = [ "otlp" ];
};
})
# scrape opentelemetry-colectors metrics
# todo: this should be collected another way (opentelemetry internal?)
# todo : enable me only when metrics.endpoint is set.
(mkIf config.components.monitor.metrics.enable {
services.opentelemetry-collector.settings = {
receivers = {
prometheus.config.scrape_configs = [
{
job_name = "otelcol";
scrape_interval = "10s";
static_configs = [{
targets = [ cfg.metrics.endpoint ];
}];
metric_relabel_configs = [
{
source_labels = [ "__name__" ];
regex = ".*grpc_io.*";
action = "drop";
}
];
}
];
};
service = {
pipelines.metrics = {
receivers = [ "prometheus" ];
};
# todo : this should be automatically be collected
# open telemetries own metrics?
telemetry.metrics.address = cfg.metrics.endpoint;
};
};
})
(mkIf (! config.components.monitor.metrics.enable) {
services.opentelemetry-collector.settings = {
service.telemetry.metrics.level = "none";
};
})
];
}