make monitoring a component and easy to configure

nixos/components/monitor/default.nix

@@ -3,7 +3,6 @@ with lib;
 with types;
 {
 
-
   options.components.monitor = {
     enable = mkOption {
       type = bool;
@@ -13,8 +12,10 @@ with types;
 
   imports = [
     ./netdata.nix
+    ./opentelemetry.nix
+    ./prometheus.nix
+    ./telegraf.nix
   ];
 
-
   config = mkIf config.components.monitor.enable { };
 }

nixos/components/monitor/opentelemetry.nix

@@ -0,0 +1,124 @@
+{ pkgs, config, lib, ... }:
+with lib;
+with types;
+let
+  cfg = config.components.monitor.opentelemetry;
+in
+{
+  options.components.monitor.opentelemetry = {
+    receiver.endpoint = mkOption {
+      type = nullOr str;
+      default = null;
+      description = "endpoint to receive the opentelementry data from other collectors";
+    };
+    exporter.endpoint = mkOption {
+      type = nullOr str;
+      default = null;
+      description = "endpoint to ship opentelementry data too";
+    };
+    metrics.endpoint = mkOption {
+      type = str;
+      default = "127.0.0.1:8100";
+      description = "endpoint on where to provide opentelementry metrics";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf config.components.monitor.enable {
+      services.opentelemetry-collector = {
+        enable = true;
+        package = pkgs.unstable.opentelemetry-collector-contrib;
+      };
+    })
+
+    # add default tags to metrics
+    # todo : make sure we filter out metrics from otlp receivers
+    (mkIf config.components.monitor.enable {
+      services.opentelemetry-collector.settings = {
+        service.pipelines.metrics.processors = [ "metricstransform" ];
+        processors.metricstransform.transforms = [
+          {
+            include = ".*";
+            match_type = "regexp";
+            action = "update";
+            operations = [{
+              action = "add_label";
+              new_label = "machine";
+              new_value = config.networking.hostName;
+            }];
+          }
+        ];
+      };
+    })
+
+    # ship to next instance
+    (mkIf (config.components.monitor.opentelemetry.exporter.endpoint != null) {
+      services.opentelemetry-collector.settings = {
+        exporters.otlp = {
+          endpoint = cfg.exporter.endpoint;
+          tls.insecure = true;
+        };
+        service = {
+          pipelines.metrics = {
+            exporters = [ "otlp" ];
+          };
+          #pipelines.logs = {
+          #  exporters = [ "otlp" ];
+          #};
+        };
+      };
+    })
+
+    # ship from other instance
+    (mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) {
+      services.opentelemetry-collector.settings = {
+        receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint;
+        service = {
+          pipelines.metrics = {
+            receivers = [ "otlp" ];
+          };
+          #pipelines.logs = {
+          #  exporters = [ "otlp" ];
+          #};
+        };
+      };
+    })
+
+    # scrape opentelemetry-colectors metrics
+    # todo: this should be collected another way (opentelemetry internal?)
+    (mkIf config.components.monitor.enable {
+      services.opentelemetry-collector.settings = {
+        receivers = {
+          prometheus.config.scrape_configs = [
+            {
+              job_name = "otelcol";
+              scrape_interval = "10s";
+              static_configs = [{
+                targets = [ cfg.metrics.endpoint ];
+              }];
+              metric_relabel_configs = [
+                {
+                  source_labels = [ "__name__" ];
+                  regex = ".*grpc_io.*";
+                  action = "drop";
+                }
+              ];
+            }
+          ];
+        };
+
+        service = {
+          pipelines.metrics = {
+            receivers = [ "prometheus" ];
+          };
+
+          # todo : this should be automatically be collected
+          # open telemetries own metrics?
+          telemetry.metrics.address = cfg.metrics.endpoint;
+        };
+
+      };
+    })
+  ];
+
+}

nixos/components/monitor/prometheus.nix

@@ -0,0 +1,45 @@
+{ config, lib, ... }:
+with lib;
+with types;
+let
+  cfg = config.components.monitor.prometheus;
+in
+{
+  options.components.monitor.prometheus = {
+    enable = mkOption {
+      type = lib.types.bool;
+      default = config.components.monitor.enable;
+    };
+    port = mkOption {
+      type = int;
+      default = 8090;
+      description = "port to provide Prometheus export";
+    };
+  };
+
+  config = mkMerge [
+
+    (mkIf config.components.monitor.prometheus.enable {
+      services.prometheus = {
+        checkConfig = "syntax-only";
+        enable = true;
+      };
+    })
+
+    (mkIf config.components.monitor.prometheus.enable {
+      services.opentelemetry-collector.settings = {
+        exporters.prometheus.endpoint = "127.0.0.1:${toString cfg.port}";
+        service.pipelines.metrics.exporters = [ "prometheus" ];
+      };
+      services.prometheus.scrapeConfigs = [
+        {
+          job_name = "opentelemetry";
+          metrics_path = "/metrics";
+          scrape_interval = "10s";
+          static_configs = [{ targets = [ "localhost:${toString cfg.port}" ]; }];
+        }
+      ];
+    })
+
+  ];
+}

nixos/components/monitor/telegraf.nix

@@ -0,0 +1,46 @@
+{ config, pkgs, lib, ... }:
+with lib;
+with types;
+let
+  cfg = config.components.monitor;
+in
+{
+  options.components.monitor = {
+    influxDBPort = mkOption {
+      type = int;
+      default = 8088;
+      description = "Port to listen on influxDB input";
+    };
+  };
+
+  config = lib.mkMerge [
+    (mkIf config.components.monitor.enable {
+      # opentelemetry wireing
+      services.opentelemetry-collector.settings = {
+        receivers.influxdb.endpoint = "127.0.0.1:${toString cfg.influxDBPort}";
+        service.pipelines.metrics.receivers = [ "influxdb" ];
+      };
+      services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString cfg.influxDBPort}" ];
+    })
+
+    (mkIf config.components.monitor.enable {
+
+      systemd.services.telegraf.path = [ pkgs.inetutils ];
+
+      services.telegraf = {
+        enable = true;
+        extraConfig = {
+          # https://github.com/influxdata/telegraf/tree/master/plugins/inputs < all them plugins
+          inputs = {
+            cpu = { };
+            diskio = { };
+            processes = { };
+            system = { };
+            systemd_units = { };
+            ping = [{ urls = [ "10.100.0.1" ]; }]; # actually important to make machine visible over wireguard
+          };
+        };
+      };
+    })
+  ];
+}