monitor containers

nixos/components/monitor/default.nix

@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ lib, config, ... }:
 with lib;
 with types;
 {
@@ -8,15 +8,23 @@ with types;
       type = bool;
       default = true;
     };
+    metrics.enable = mkOption {
+      type = bool;
+      default = config.components.monitor.enable;
+    };
+    logs.enable = mkOption {
+      type = bool;
+      default = config.components.monitor.enable;
+    };
   };
 
   imports = [
-    ./export-zfs.nix
-    ./netdata.nix
+    ./logs-promtail.nix
+    ./metrics-export-zfs.nix
+    ./metrics-netdata.nix
+    ./metrics-prometheus.nix
+    ./metrics-telegraf.nix
     ./opentelemetry.nix
-    ./prometheus.nix
-    ./promtail.nix
-    ./telegraf.nix
   ];
 
   config = mkIf config.components.monitor.enable { };

nixos/components/monitor/logs-promtail.nix

@@ -8,7 +8,7 @@ in
   options.components.monitor.promtail = {
     enable = mkOption {
       type = lib.types.bool;
-      default = config.components.monitor.enable;
+      default = config.components.monitor.logs.enable;
     };
     port = mkOption {
       type = int;
@@ -32,10 +32,7 @@ in
       services.promtail = {
         enable = true;
         configuration = {
-          server = {
-            http_listen_port = 28183;
-            grpc_listen_port = 0;
-          };
+          server. disable = true;
           positions.filename = "/var/cache/promtail/positions.yaml";
 
           clients = [

nixos/components/monitor/metrics-export-zfs.nix

@@ -4,7 +4,7 @@ with types;
 {
   options.components.monitor.exporters.zfs.enable = mkOption {
     type = lib.types.bool;
-    default = config.components.monitor.enable;
+    default = config.components.monitor.metrics.enable;
   };
 
   config = mkMerge [

nixos/components/monitor/metrics-netdata.nix

@@ -2,8 +2,14 @@
 with lib;
 with types;
 {
+  options.components.monitor.netdata = {
+    enable = mkOption {
+      type = bool;
+      default = config.components.monitor.metrics.enable;
+    };
+  };
 
-  config = lib.mkIf config.components.monitor.enable {
+  config = mkIf config.components.monitor.netdata.enable {
 
     # netdata sink
     services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [

nixos/components/monitor/metrics-prometheus.nix

@@ -8,7 +8,7 @@ in
   options.components.monitor.prometheus = {
     enable = mkOption {
       type = lib.types.bool;
-      default = config.components.monitor.enable;
+      default = config.components.monitor.metrics.enable;
     };
     port = mkOption {
       type = int;

nixos/components/monitor/metrics-telegraf.nix

@@ -2,10 +2,14 @@
 with lib;
 with types;
 let
-  cfg = config.components.monitor;
+  cfg = config.components.monitor.telegraf;
 in
 {
-  options.components.monitor = {
+  options.components.monitor.telegraf = {
+    enable = mkOption {
+      type = lib.types.bool;
+      default = config.components.monitor.metrics.enable;
+    };
     influxDBPort = mkOption {
       type = int;
       default = 8088;
@@ -14,7 +18,7 @@ in
   };
 
   config = lib.mkMerge [
-    (mkIf config.components.monitor.enable {
+    (mkIf config.components.monitor.telegraf.enable {
       # opentelemetry wireing
       services.opentelemetry-collector.settings = {
         receivers.influxdb.endpoint = "127.0.0.1:${toString cfg.influxDBPort}";
@@ -23,7 +27,7 @@ in
       services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString cfg.influxDBPort}" ];
     })
 
-    (mkIf config.components.monitor.enable {
+    (mkIf config.components.monitor.telegraf.enable {
 
       systemd.services.telegraf.path = [ pkgs.inetutils ];
 

nixos/components/monitor/opentelemetry.nix

@@ -29,6 +29,7 @@ in
   };
 
   config = mkMerge [
+
     (mkIf config.components.monitor.enable {
       services.opentelemetry-collector = {
         enable = true;
@@ -41,9 +42,6 @@ in
     (mkIf config.components.monitor.enable {
       services.opentelemetry-collector.settings = {
 
-        service.pipelines.metrics.processors = [ "metricstransform" "resourcedetection/system" ];
-        service.pipelines.logs.processors = [ "resourcedetection/system" ];
-
         processors = {
 
           # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md
@@ -68,6 +66,20 @@ in
         };
       };
     })
+    (mkIf config.components.monitor.metrics.enable {
+      services.opentelemetry-collector.settings = {
+        service.pipelines.metrics.processors = [
+          "metricstransform"
+          "resourcedetection/system"
+        ];
+      };
+    })
+    (mkIf config.components.monitor.logs.enable {
+      services.opentelemetry-collector.settings = {
+        service.pipelines.logs.processors = [ "resourcedetection/system" ];
+      };
+    })
+
 
     (mkIf (config.components.monitor.opentelemetry.exporter.debug != null) {
       services.opentelemetry-collector.settings = {
@@ -90,35 +102,62 @@ in
           endpoint = cfg.exporter.endpoint;
           tls.insecure = true;
         };
-        service = {
-          pipelines.metrics = {
-            exporters = [ "otlp" ];
-          };
-          pipelines.logs = {
-            exporters = [ "otlp" ];
-          };
-        };
       };
     })
+    (mkIf
+      (
+        config.components.monitor.opentelemetry.exporter.endpoint != null &&
+        config.components.monitor.logs.enable
+      )
+      {
+        services.opentelemetry-collector.settings = {
+          service.pipelines.logs.exporters = [ "otlp" ];
+        };
+      })
+    (mkIf
+      (
+        config.components.monitor.opentelemetry.exporter.endpoint != null &&
+        config.components.monitor.metrics.enable
+      )
+      {
+        services.opentelemetry-collector.settings = {
+          service.pipelines.metrics.exporters = [ "otlp" ];
+        };
+      })
 
     # ship from other instance
     (mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) {
       services.opentelemetry-collector.settings = {
         receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint;
-        service = {
-          pipelines.metrics = {
-            receivers = [ "otlp" ];
-          };
-          pipelines.logs = {
-            receivers = [ "otlp" ];
-          };
-        };
       };
     })
+    (mkIf
+      (
+        config.components.monitor.opentelemetry.receiver.endpoint != null &&
+        config.components.monitor.logs.enable
+      )
+      {
+        services.opentelemetry-collector.settings = {
+          service.pipelines.logs.receivers = [ "otlp" ];
+        };
+      })
+    (mkIf
+      (
+        config.components.monitor.opentelemetry.receiver.endpoint != null &&
+        config.components.monitor.metrics.enable
+      )
+      {
+        services.opentelemetry-collector.settings = {
+          service.pipelines.metrics.receivers = [ "otlp" ];
+        };
+      })
+
+
 
     # scrape opentelemetry-colectors metrics
     # todo: this should be collected another way (opentelemetry internal?)
-    (mkIf config.components.monitor.enable {
+    # todo : enable me only when metrics.endpoint is set.
+    (mkIf config.components.monitor.metrics.enable {
       services.opentelemetry-collector.settings = {
         receivers = {
           prometheus.config.scrape_configs = [
@@ -151,6 +190,11 @@ in
 
       };
     })
+    (mkIf (! config.components.monitor.metrics.enable) {
+      services.opentelemetry-collector.settings = {
+        service.telemetry.metrics.level = "none";
+      };
+    })
   ];
 
 }

nixos/machines/chungus/telemetry/opentelemetry-hass.nix

@@ -29,7 +29,7 @@
         metrics_path = "/api/prometheus";
         bearer_token_file = toString config.sops.secrets.hass_long_term_token.path;
         static_configs = [{
-          targets = [ "localhost:8123" ];
+          targets = [ "127.0.0.1:8123" ];
         }];
       }
     ];

nixos/machines/orbi/container-monitoring.nix

@@ -0,0 +1,22 @@
+{ lib, config, ... }:
+with lib;
+with types;
+{
+  imports = [
+    ../../components/monitor
+  ];
+
+  # todo create a the components.monitor.container module.
+  options.promtail.port = mkOption {
+    type = int;
+  };
+
+  config = {
+    components.monitor.enable = true;
+    components.monitor.metrics.enable = false;
+    components.monitor.promtail.port = config.promtail.port;
+    components.monitor.opentelemetry.exporter.endpoint = "127.0.0.1:4317";
+  };
+
+
+}

nixos/machines/orbi/media-nextcloud.nix

@@ -86,14 +86,18 @@ in
     privateNetwork = false;
     autoStart = true;
 
-    config = { config, pkgs, lib, ... }: {
+
+    config = { config, lib, ... }: {
+      nixpkgs.pkgs = pkgs;
+      imports = [ ./container-monitoring.nix ];
+      promtail.port = 3502;
+      system.stateVersion = "23.11";
 
       # Configuring nameservers for containers is currently broken.
       # Therefore in some cases internet connectivity can be broken inside the containers.
       # A temporary workaround is to manually write the /etc/nixos/resolv.conf file like this:
       #environment.etc."resolv.conf".text = "nameserver 8.8.8.8";
 
-      system.stateVersion = "23.11";
 
       users.users.nextcloud.uid = nextcloudUid;
 

nixos/machines/orbi/media-tdarr.nix

@@ -1,54 +0,0 @@
-{ config, lib, pkgs, ... }:
-{
-
-  # https://docs.tdarr.io/docs/installation/docker/run-compose
-  virtualisation.oci-containers = {
-    containers.tdarr = {
-      volumes = [
-        "/media/arr/tdarr/server:/app/server"
-        "/media/arr/tdarr/configs:/app/configs"
-        "/media/arr/tdarr/logs:/app/logs"
-        "/media/arr/tdarr/transcode_cache:/temp"
-        "/media:/media"
-      ];
-      environment = {
-        serverIP = "0.0.0.0";
-        serverPort = "8266";
-        webUIPort = "8265";
-        internalNode = "true";
-        inContainer = "true";
-        nodeName = "robi";
-        TZ = "Europe/Berlin";
-        PUID = toString config.users.users.media.uid;
-        PGID = toString config.users.groups.media.gid;
-      };
-      ports = [
-        "127.0.0.1:8265:8265" # WebUI
-        # "8266:8266" # server port
-      ];
-      image = "ghcr.io/haveagitgat/tdarr:latest"; # Warning: if the tag does not change, the image will not be updated
-      extraOptions = [
-        #"--network=bridge"
-        #"--privileged"
-      ];
-    };
-  };
-
-  #networking.firewall.interfaces.wg0.allowedTCPPorts = [ 8266 ];
-  #networking.firewall.interfaces.wg0.allowedUDPPorts = [ 8266 ];
-
-  #networking.firewall.interfaces.enp0s31f6.allowedTCPPorts = [ 8266 ];
-  #networking.firewall.interfaces.enp0s31f6.allowedUDPPorts = [ 8266 ];
-
-  services.nginx.virtualHosts."tdarr.${config.networking.hostName}.private" = {
-    extraConfig = ''
-      allow ${config.tinc.private.subnet};
-      deny all;
-    '';
-    locations."/" = {
-      proxyPass = "http://localhost:8265";
-      proxyWebsockets = true;
-    };
-  };
-
-}

nixos/machines/orbi/service-photoprism.nix

@@ -13,7 +13,10 @@ in
     privateNetwork = false;
     autoStart = true;
 
-    config = { config, pkgs, lib, ... }: {
+    config = { config, lib, ... }: {
+      nixpkgs.pkgs = pkgs;
+      imports = [ ./container-monitoring.nix ];
+      promtail.port = 3503;
       system.stateVersion = "23.11";
 
       # Photoprism

nixos/machines/orbi/social-matrix-terranix.nix

@@ -68,7 +68,10 @@ in
       };
     };
 
-    config = { config, pkgs, lib, ... }: {
+    config = { config, lib, ... }: {
+      nixpkgs.pkgs = pkgs;
+      imports = [ ./container-monitoring.nix ];
+      promtail.port = 3504;
       system.stateVersion = "23.11";
 
       services.postgresql = {