From 6f527173d958e84d8465a8bb4123f66e9f04ec47 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Wed, 15 May 2024 01:11:06 +0200 Subject: [PATCH] default tags right is done by opentelemetry now --- nixos/components/monitor/netdata.nix | 28 ++++------ .../cherry/telemetry/opentelemetry.nix | 55 ++++++------------- .../machines/cherry/telemetry/prometheus.nix | 1 - nixos/machines/cherry/telemetry/telegraf.nix | 18 ++++-- 4 files changed, 41 insertions(+), 61 deletions(-) diff --git a/nixos/components/monitor/netdata.nix b/nixos/components/monitor/netdata.nix index a5a916b..b6f801d 100644 --- a/nixos/components/monitor/netdata.nix +++ b/nixos/components/monitor/netdata.nix @@ -5,29 +5,25 @@ with types; config = lib.mkIf config.components.monitor.enable { + # netdata sink + services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [ + { + job_name = "netdata"; + scrape_interval = "10s"; + metrics_path = "/api/v1/allmetrics"; + params.format = [ "prometheus" ]; + static_configs = [{ targets = [ "127.0.0.1:19999" ]; }]; + } + ]; + + # https://docs.netdata.cloud/daemon/config/ services.netdata = { enable = lib.mkDefault true; - # https://docs.netdata.cloud/daemon/config/ config = { global = { "memory mode" = "ram"; }; }; - #configDir."python.d.conf" = pkgs.writeText "python.d.conf" '' - # example: yes - # default_run: no - # samba: yes - #''; }; - - # add samba to path of python plugin - #systemd.services.netdata.path = [ pkgs.sudo pkgs.samba ]; - #systemd.services.netdata.serviceConfig.CapabilityBoundingSet = [ "~" ]; - #security.sudo.extraConfig = '' - # netdata ALL=(root) NOPASSWD: ${pkgs.samba}/bin/smbstatus - # netdata ALL=(root) NOPASSWD: /run/current-system/sw/bin/smbstatus - #''; - - }; } diff --git a/nixos/machines/cherry/telemetry/opentelemetry.nix b/nixos/machines/cherry/telemetry/opentelemetry.nix index c9b4322..7eb12f6 100644 --- a/nixos/machines/cherry/telemetry/opentelemetry.nix +++ b/nixos/machines/cherry/telemetry/opentelemetry.nix @@ -1,19 +1,10 @@ { pkgs, config, ... }: let - telegraf_sink = 8088; prometheus_port = 8090; in { imports = [ - # telemetry sink - { - services.opentelemetry-collector.settings = { - receivers.influxdb.endpoint = "127.0.0.1:${toString telegraf_sink }"; - service.pipelines.metrics.receivers = [ "influxdb" ]; - }; - services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString telegraf_sink}" ]; - } # prometheus export { @@ -31,31 +22,27 @@ in ]; } - # todo : move to netdata component - # netdata sink - { - services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [ - { - job_name = "netdata"; - scrape_interval = "10s"; - metrics_path = "/api/v1/allmetrics"; - params.format = [ "prometheus" ]; - static_configs = [{ - targets = [ "127.0.0.1:19999" ]; - labels = { - service = "netdata"; - server = config.networking.hostName; - }; - }]; - } - ]; - } ]; services.opentelemetry-collector = { enable = true; package = pkgs.unstable.opentelemetry-collector-contrib; settings = { + # add default tags + # todo : make sure we filter out metrics from otlp receivers + processors.metricstransform.transforms = [ + { + include = ".*"; + match_type = "regexp"; + action = "update"; + operations = [{ + action = "add_label"; + new_label = "server"; + new_value = config.networking.hostName; + }]; + } + ]; + receivers = { # scrape opentelemetry-colectors metrics prometheus.config.scrape_configs = [ @@ -65,10 +52,6 @@ in scrape_interval = "10s"; static_configs = [{ targets = [ "127.0.0.1:8100" ]; - labels = { - service = "otelcol"; - server = config.networking.hostName; - }; }]; metric_relabel_configs = [ { @@ -80,14 +63,9 @@ in } { - job_name = "node"; + job_name = "node-exporter"; static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; - labels = { - service = "node-exporter"; - # todo : use a processor for this - server = config.networking.hostName; - }; }]; } ]; @@ -103,6 +81,7 @@ in service = { pipelines.metrics = { receivers = [ "prometheus" ]; + processors = [ "metricstransform" ]; exporters = [ "otlp" ]; }; diff --git a/nixos/machines/cherry/telemetry/prometheus.nix b/nixos/machines/cherry/telemetry/prometheus.nix index 7fbd163..24d5e71 100644 --- a/nixos/machines/cherry/telemetry/prometheus.nix +++ b/nixos/machines/cherry/telemetry/prometheus.nix @@ -17,7 +17,6 @@ services.prometheus = { checkConfig = "syntax-only"; enable = true; - exporters = { node = { enable = true; diff --git a/nixos/machines/cherry/telemetry/telegraf.nix b/nixos/machines/cherry/telemetry/telegraf.nix index 863b2ac..7c9d77f 100644 --- a/nixos/machines/cherry/telemetry/telegraf.nix +++ b/nixos/machines/cherry/telemetry/telegraf.nix @@ -1,17 +1,23 @@ { config, pkgs, ... }: +let + telegraf_sink = 8088; +in { + imports = [ + { + services.opentelemetry-collector.settings = { + receivers.influxdb.endpoint = "127.0.0.1:${toString telegraf_sink }"; + service.pipelines.metrics.receivers = [ "influxdb" ]; + }; + services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString telegraf_sink}" ]; + } + ]; systemd.services.telegraf.path = [ pkgs.inetutils ]; services.telegraf = { enable = true; extraConfig = { - - global_tags = { - service = "telegraf"; - server = config.networking.hostName; - }; - # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins inputs = { cpu = { };