default tags right is done by opentelemetry now

nixos/components/monitor/netdata.nix

@@ -5,29 +5,25 @@ with types;
 
   config = lib.mkIf config.components.monitor.enable {
 
+    # netdata sink
+    services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [
+      {
+        job_name = "netdata";
+        scrape_interval = "10s";
+        metrics_path = "/api/v1/allmetrics";
+        params.format = [ "prometheus" ];
+        static_configs = [{ targets = [ "127.0.0.1:19999" ]; }];
+      }
+    ];
+
+    # https://docs.netdata.cloud/daemon/config/
     services.netdata = {
       enable = lib.mkDefault true;
-      # https://docs.netdata.cloud/daemon/config/
       config = {
         global = {
           "memory mode" = "ram";
         };
       };
-      #configDir."python.d.conf" = pkgs.writeText "python.d.conf" ''
-      #  example: yes
-      #  default_run: no
-      #  samba: yes
-      #'';
     };
-
-    # add samba to path of python plugin
-    #systemd.services.netdata.path = [ pkgs.sudo pkgs.samba ];
-    #systemd.services.netdata.serviceConfig.CapabilityBoundingSet = [ "~" ];
-    #security.sudo.extraConfig = ''
-    #  netdata ALL=(root) NOPASSWD: ${pkgs.samba}/bin/smbstatus
-    #  netdata ALL=(root) NOPASSWD: /run/current-system/sw/bin/smbstatus
-    #'';
-
-
   };
 }

nixos/machines/cherry/telemetry/opentelemetry.nix

@@ -1,19 +1,10 @@
 { pkgs, config, ... }:
 let
-  telegraf_sink = 8088;
   prometheus_port = 8090;
 in
 {
 
   imports = [
-    # telemetry sink
-    {
-      services.opentelemetry-collector.settings = {
-        receivers.influxdb.endpoint = "127.0.0.1:${toString telegraf_sink }";
-        service.pipelines.metrics.receivers = [ "influxdb" ];
-      };
-      services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString telegraf_sink}" ];
-    }
 
     # prometheus export
     {
@@ -31,31 +22,27 @@ in
       ];
     }
 
-    # todo : move to netdata component
-    # netdata sink
-    {
-      services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [
-        {
-          job_name = "netdata";
-          scrape_interval = "10s";
-          metrics_path = "/api/v1/allmetrics";
-          params.format = [ "prometheus" ];
-          static_configs = [{
-            targets = [ "127.0.0.1:19999" ];
-            labels = {
-              service = "netdata";
-              server = config.networking.hostName;
-            };
-          }];
-        }
-      ];
-    }
   ];
 
   services.opentelemetry-collector = {
     enable = true;
     package = pkgs.unstable.opentelemetry-collector-contrib;
     settings = {
+      # add default tags
+      # todo : make sure we filter out metrics from otlp receivers
+      processors.metricstransform.transforms = [
+        {
+          include = ".*";
+          match_type = "regexp";
+          action = "update";
+          operations = [{
+            action = "add_label";
+            new_label = "server";
+            new_value = config.networking.hostName;
+          }];
+        }
+      ];
+
       receivers = {
         # scrape opentelemetry-colectors metrics
         prometheus.config.scrape_configs = [
@@ -65,10 +52,6 @@ in
             scrape_interval = "10s";
             static_configs = [{
               targets = [ "127.0.0.1:8100" ];
-              labels = {
-                service = "otelcol";
-                server = config.networking.hostName;
-              };
             }];
             metric_relabel_configs = [
               {
@@ -80,14 +63,9 @@ in
           }
 
           {
-            job_name = "node";
+            job_name = "node-exporter";
             static_configs = [{
               targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
-              labels = {
-                service = "node-exporter";
-                # todo : use a processor for this
-                server = config.networking.hostName;
-              };
             }];
           }
         ];
@@ -103,6 +81,7 @@ in
       service = {
         pipelines.metrics = {
           receivers = [ "prometheus" ];
+          processors = [ "metricstransform" ];
           exporters = [ "otlp" ];
         };
 

nixos/machines/cherry/telemetry/prometheus.nix

@@ -17,7 +17,6 @@
   services.prometheus = {
     checkConfig = "syntax-only";
     enable = true;
-
     exporters = {
       node = {
         enable = true;

nixos/machines/cherry/telemetry/telegraf.nix

@@ -1,17 +1,23 @@
 { config, pkgs, ... }:
+let
+  telegraf_sink = 8088;
+in
 {
+  imports = [
+    {
+      services.opentelemetry-collector.settings = {
+        receivers.influxdb.endpoint = "127.0.0.1:${toString telegraf_sink }";
+        service.pipelines.metrics.receivers = [ "influxdb" ];
+      };
+      services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString telegraf_sink}" ];
+    }
+  ];
 
   systemd.services.telegraf.path = [ pkgs.inetutils ];
 
   services.telegraf = {
     enable = true;
     extraConfig = {
-
-      global_tags = {
-        service = "telegraf";
-        server = config.networking.hostName;
-      };
-
       # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins
       inputs = {
         cpu = { };