add opentelemetry everywhere

nixos/machines/chungus/configuration.nix

@@ -37,10 +37,12 @@
     # logging
     ./loki.nix
     ./loki-promtail.nix
-    ./prometheus.nix
     ./grafana.nix
-    ./telegraf.nix
-    ./telegraf-smart.nix
+
+    ./telemetry/opentelemetry.nix
+    ./telemetry/prometheus.nix
+    ./telemetry/telegraf.nix
+    ./telemetry/telegraf-smart.nix
 
     #./home-display.nix
 

nixos/machines/chungus/telemetry/opentelemetry.nix

@@ -0,0 +1,58 @@
+{ pkgs, ... }:
+{
+  services.opentelemetry-collector = {
+    enable = true;
+    package = pkgs.unstable.opentelemetry-collector-contrib;
+    settings = {
+      receivers = {
+
+        # receive metrics from other open telemetry collectors
+        otlp.protocols.grpc.endpoint = "0.0.0.0:4317";
+
+        # provide a influxdb sink
+        influxdb = {
+          endpoint = "127.0.0.1:8088";
+        };
+
+        # scrape opentelemetry-colectors metrics
+        prometheus.config.scrape_configs = [
+          {
+            job_name = "netdata";
+            scrape_interval = "10s";
+            metrics_path = "/api/v1/allmetrics";
+            params.format = [ "prometheus" ];
+            static_configs = [{ targets = [ "127.0.0.1:19999" ]; }];
+          }
+          {
+            job_name = "otelcol";
+            scrape_interval = "10s";
+            static_configs = [{ targets = [ "127.0.0.1:8100" ]; }];
+            metric_relabel_configs = [
+              {
+                source_labels = [ "__name__" ];
+                regex = ".*grpc_io.*";
+                action = "drop";
+              }
+            ];
+          }
+        ];
+      };
+
+      exporters = {
+        # provide prometheus sink under `/metrics` to 
+        prometheus = {
+          endpoint = "127.0.0.1:8090";
+        };
+      };
+
+      service = {
+        pipelines.metrics = {
+          receivers = [ "otlp" "influxdb" "prometheus" ];
+          exporters = [ "prometheus" ];
+        };
+        # open telemetries own metrics?
+        telemetry.metrics.address = "0.0.0.0:8100";
+      };
+    };
+  };
+}

nixos/machines/chungus/telemetry/prometheus.nix

@@ -59,15 +59,6 @@
       }))
     ];
 
-
-
-
-    #alertmanager = {
-    #  enable = true;
-    #  configuration = {
-    #};
-    #};
-
     exporters = {
       node = {
         enable = true;

nixos/machines/orbi/configuration.nix

@@ -41,9 +41,9 @@
     #./loki.nix
     #./loki-promtail.nix
     #./grafana.nix
-    ./system-telegraf.nix
-    ./system-opentelemetry.nix
-    #./system-prometheus.nix
+    ./telemetry/telegraf.nix
+    ./telemetry/opentelemetry.nix
+    #./telemetry/prometheus.nix
 
     #./cache.nix
   ];