diff --git a/nixos/components/monitor/default.nix b/nixos/components/monitor/default.nix index a7fe06a..d8007eb 100644 --- a/nixos/components/monitor/default.nix +++ b/nixos/components/monitor/default.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, config, ... }: with lib; with types; { @@ -8,15 +8,23 @@ with types; type = bool; default = true; }; + metrics.enable = mkOption { + type = bool; + default = config.components.monitor.enable; + }; + logs.enable = mkOption { + type = bool; + default = config.components.monitor.enable; + }; }; imports = [ - ./export-zfs.nix - ./netdata.nix + ./logs-promtail.nix + ./metrics-export-zfs.nix + ./metrics-netdata.nix + ./metrics-prometheus.nix + ./metrics-telegraf.nix ./opentelemetry.nix - ./prometheus.nix - ./promtail.nix - ./telegraf.nix ]; config = mkIf config.components.monitor.enable { }; diff --git a/nixos/components/monitor/promtail.nix b/nixos/components/monitor/logs-promtail.nix similarity index 98% rename from nixos/components/monitor/promtail.nix rename to nixos/components/monitor/logs-promtail.nix index 986e546..7ef67fd 100644 --- a/nixos/components/monitor/promtail.nix +++ b/nixos/components/monitor/logs-promtail.nix @@ -8,7 +8,7 @@ in options.components.monitor.promtail = { enable = mkOption { type = lib.types.bool; - default = config.components.monitor.enable; + default = config.components.monitor.logs.enable; }; port = mkOption { type = int; @@ -32,10 +32,7 @@ in services.promtail = { enable = true; configuration = { - server = { - http_listen_port = 28183; - grpc_listen_port = 0; - }; + server. disable = true; positions.filename = "/var/cache/promtail/positions.yaml"; clients = [ diff --git a/nixos/components/monitor/export-zfs.nix b/nixos/components/monitor/metrics-export-zfs.nix similarity index 93% rename from nixos/components/monitor/export-zfs.nix rename to nixos/components/monitor/metrics-export-zfs.nix index c416faf..63892ae 100644 --- a/nixos/components/monitor/export-zfs.nix +++ b/nixos/components/monitor/metrics-export-zfs.nix @@ -4,7 +4,7 @@ with types; { options.components.monitor.exporters.zfs.enable = mkOption { type = lib.types.bool; - default = config.components.monitor.enable; + default = config.components.monitor.metrics.enable; }; config = mkMerge [ diff --git a/nixos/components/monitor/netdata.nix b/nixos/components/monitor/metrics-netdata.nix similarity index 74% rename from nixos/components/monitor/netdata.nix rename to nixos/components/monitor/metrics-netdata.nix index b6f801d..6132405 100644 --- a/nixos/components/monitor/netdata.nix +++ b/nixos/components/monitor/metrics-netdata.nix @@ -2,8 +2,14 @@ with lib; with types; { + options.components.monitor.netdata = { + enable = mkOption { + type = bool; + default = config.components.monitor.metrics.enable; + }; + }; - config = lib.mkIf config.components.monitor.enable { + config = mkIf config.components.monitor.netdata.enable { # netdata sink services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs = [ diff --git a/nixos/components/monitor/prometheus.nix b/nixos/components/monitor/metrics-prometheus.nix similarity index 94% rename from nixos/components/monitor/prometheus.nix rename to nixos/components/monitor/metrics-prometheus.nix index 52e5e69..095bc59 100644 --- a/nixos/components/monitor/prometheus.nix +++ b/nixos/components/monitor/metrics-prometheus.nix @@ -8,7 +8,7 @@ in options.components.monitor.prometheus = { enable = mkOption { type = lib.types.bool; - default = config.components.monitor.enable; + default = config.components.monitor.metrics.enable; }; port = mkOption { type = int; diff --git a/nixos/components/monitor/telegraf.nix b/nixos/components/monitor/metrics-telegraf.nix similarity index 78% rename from nixos/components/monitor/telegraf.nix rename to nixos/components/monitor/metrics-telegraf.nix index c56bb85..ab04632 100644 --- a/nixos/components/monitor/telegraf.nix +++ b/nixos/components/monitor/metrics-telegraf.nix @@ -2,10 +2,14 @@ with lib; with types; let - cfg = config.components.monitor; + cfg = config.components.monitor.telegraf; in { - options.components.monitor = { + options.components.monitor.telegraf = { + enable = mkOption { + type = lib.types.bool; + default = config.components.monitor.metrics.enable; + }; influxDBPort = mkOption { type = int; default = 8088; @@ -14,7 +18,7 @@ in }; config = lib.mkMerge [ - (mkIf config.components.monitor.enable { + (mkIf config.components.monitor.telegraf.enable { # opentelemetry wireing services.opentelemetry-collector.settings = { receivers.influxdb.endpoint = "127.0.0.1:${toString cfg.influxDBPort}"; @@ -23,7 +27,7 @@ in services.telegraf.extraConfig.outputs.influxdb_v2.urls = [ "http://127.0.0.1:${toString cfg.influxDBPort}" ]; }) - (mkIf config.components.monitor.enable { + (mkIf config.components.monitor.telegraf.enable { systemd.services.telegraf.path = [ pkgs.inetutils ]; diff --git a/nixos/components/monitor/opentelemetry.nix b/nixos/components/monitor/opentelemetry.nix index 0878fa1..5f4cf0b 100644 --- a/nixos/components/monitor/opentelemetry.nix +++ b/nixos/components/monitor/opentelemetry.nix @@ -29,6 +29,7 @@ in }; config = mkMerge [ + (mkIf config.components.monitor.enable { services.opentelemetry-collector = { enable = true; @@ -41,9 +42,6 @@ in (mkIf config.components.monitor.enable { services.opentelemetry-collector.settings = { - service.pipelines.metrics.processors = [ "metricstransform" "resourcedetection/system" ]; - service.pipelines.logs.processors = [ "resourcedetection/system" ]; - processors = { # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md @@ -68,6 +66,20 @@ in }; }; }) + (mkIf config.components.monitor.metrics.enable { + services.opentelemetry-collector.settings = { + service.pipelines.metrics.processors = [ + "metricstransform" + "resourcedetection/system" + ]; + }; + }) + (mkIf config.components.monitor.logs.enable { + services.opentelemetry-collector.settings = { + service.pipelines.logs.processors = [ "resourcedetection/system" ]; + }; + }) + (mkIf (config.components.monitor.opentelemetry.exporter.debug != null) { services.opentelemetry-collector.settings = { @@ -90,35 +102,62 @@ in endpoint = cfg.exporter.endpoint; tls.insecure = true; }; - service = { - pipelines.metrics = { - exporters = [ "otlp" ]; - }; - pipelines.logs = { - exporters = [ "otlp" ]; - }; - }; }; }) + (mkIf + ( + config.components.monitor.opentelemetry.exporter.endpoint != null && + config.components.monitor.logs.enable + ) + { + services.opentelemetry-collector.settings = { + service.pipelines.logs.exporters = [ "otlp" ]; + }; + }) + (mkIf + ( + config.components.monitor.opentelemetry.exporter.endpoint != null && + config.components.monitor.metrics.enable + ) + { + services.opentelemetry-collector.settings = { + service.pipelines.metrics.exporters = [ "otlp" ]; + }; + }) # ship from other instance (mkIf (config.components.monitor.opentelemetry.receiver.endpoint != null) { services.opentelemetry-collector.settings = { receivers.otlp.protocols.grpc.endpoint = cfg.receiver.endpoint; - service = { - pipelines.metrics = { - receivers = [ "otlp" ]; - }; - pipelines.logs = { - receivers = [ "otlp" ]; - }; - }; }; }) + (mkIf + ( + config.components.monitor.opentelemetry.receiver.endpoint != null && + config.components.monitor.logs.enable + ) + { + services.opentelemetry-collector.settings = { + service.pipelines.logs.receivers = [ "otlp" ]; + }; + }) + (mkIf + ( + config.components.monitor.opentelemetry.receiver.endpoint != null && + config.components.monitor.metrics.enable + ) + { + services.opentelemetry-collector.settings = { + service.pipelines.metrics.receivers = [ "otlp" ]; + }; + }) + + # scrape opentelemetry-colectors metrics # todo: this should be collected another way (opentelemetry internal?) - (mkIf config.components.monitor.enable { + # todo : enable me only when metrics.endpoint is set. + (mkIf config.components.monitor.metrics.enable { services.opentelemetry-collector.settings = { receivers = { prometheus.config.scrape_configs = [ @@ -151,6 +190,11 @@ in }; }) + (mkIf (! config.components.monitor.metrics.enable) { + services.opentelemetry-collector.settings = { + service.telemetry.metrics.level = "none"; + }; + }) ]; } diff --git a/nixos/machines/chungus/telemetry/opentelemetry-hass.nix b/nixos/machines/chungus/telemetry/opentelemetry-hass.nix index ec8aff0..a57bec6 100644 --- a/nixos/machines/chungus/telemetry/opentelemetry-hass.nix +++ b/nixos/machines/chungus/telemetry/opentelemetry-hass.nix @@ -29,7 +29,7 @@ metrics_path = "/api/prometheus"; bearer_token_file = toString config.sops.secrets.hass_long_term_token.path; static_configs = [{ - targets = [ "localhost:8123" ]; + targets = [ "127.0.0.1:8123" ]; }]; } ]; diff --git a/nixos/machines/orbi/container-monitoring.nix b/nixos/machines/orbi/container-monitoring.nix new file mode 100644 index 0000000..d9793b6 --- /dev/null +++ b/nixos/machines/orbi/container-monitoring.nix @@ -0,0 +1,22 @@ +{ lib, config, ... }: +with lib; +with types; +{ + imports = [ + ../../components/monitor + ]; + + # todo create a the components.monitor.container module. + options.promtail.port = mkOption { + type = int; + }; + + config = { + components.monitor.enable = true; + components.monitor.metrics.enable = false; + components.monitor.promtail.port = config.promtail.port; + components.monitor.opentelemetry.exporter.endpoint = "127.0.0.1:4317"; + }; + + +} diff --git a/nixos/machines/orbi/media-nextcloud.nix b/nixos/machines/orbi/media-nextcloud.nix index 8af0423..5444f8f 100644 --- a/nixos/machines/orbi/media-nextcloud.nix +++ b/nixos/machines/orbi/media-nextcloud.nix @@ -86,14 +86,18 @@ in privateNetwork = false; autoStart = true; - config = { config, pkgs, lib, ... }: { + + config = { config, lib, ... }: { + nixpkgs.pkgs = pkgs; + imports = [ ./container-monitoring.nix ]; + promtail.port = 3502; + system.stateVersion = "23.11"; # Configuring nameservers for containers is currently broken. # Therefore in some cases internet connectivity can be broken inside the containers. # A temporary workaround is to manually write the /etc/nixos/resolv.conf file like this: #environment.etc."resolv.conf".text = "nameserver 8.8.8.8"; - system.stateVersion = "23.11"; users.users.nextcloud.uid = nextcloudUid; diff --git a/nixos/machines/orbi/media-tdarr.nix b/nixos/machines/orbi/media-tdarr.nix deleted file mode 100644 index 3249efa..0000000 --- a/nixos/machines/orbi/media-tdarr.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ config, lib, pkgs, ... }: -{ - - # https://docs.tdarr.io/docs/installation/docker/run-compose - virtualisation.oci-containers = { - containers.tdarr = { - volumes = [ - "/media/arr/tdarr/server:/app/server" - "/media/arr/tdarr/configs:/app/configs" - "/media/arr/tdarr/logs:/app/logs" - "/media/arr/tdarr/transcode_cache:/temp" - "/media:/media" - ]; - environment = { - serverIP = "0.0.0.0"; - serverPort = "8266"; - webUIPort = "8265"; - internalNode = "true"; - inContainer = "true"; - nodeName = "robi"; - TZ = "Europe/Berlin"; - PUID = toString config.users.users.media.uid; - PGID = toString config.users.groups.media.gid; - }; - ports = [ - "127.0.0.1:8265:8265" # WebUI - # "8266:8266" # server port - ]; - image = "ghcr.io/haveagitgat/tdarr:latest"; # Warning: if the tag does not change, the image will not be updated - extraOptions = [ - #"--network=bridge" - #"--privileged" - ]; - }; - }; - - #networking.firewall.interfaces.wg0.allowedTCPPorts = [ 8266 ]; - #networking.firewall.interfaces.wg0.allowedUDPPorts = [ 8266 ]; - - #networking.firewall.interfaces.enp0s31f6.allowedTCPPorts = [ 8266 ]; - #networking.firewall.interfaces.enp0s31f6.allowedUDPPorts = [ 8266 ]; - - services.nginx.virtualHosts."tdarr.${config.networking.hostName}.private" = { - extraConfig = '' - allow ${config.tinc.private.subnet}; - deny all; - ''; - locations."/" = { - proxyPass = "http://localhost:8265"; - proxyWebsockets = true; - }; - }; - -} diff --git a/nixos/machines/orbi/service-photoprism.nix b/nixos/machines/orbi/service-photoprism.nix index 5bb486f..e7e6233 100644 --- a/nixos/machines/orbi/service-photoprism.nix +++ b/nixos/machines/orbi/service-photoprism.nix @@ -13,7 +13,10 @@ in privateNetwork = false; autoStart = true; - config = { config, pkgs, lib, ... }: { + config = { config, lib, ... }: { + nixpkgs.pkgs = pkgs; + imports = [ ./container-monitoring.nix ]; + promtail.port = 3503; system.stateVersion = "23.11"; # Photoprism diff --git a/nixos/machines/orbi/social-matrix-terranix.nix b/nixos/machines/orbi/social-matrix-terranix.nix index 78789a7..56eb448 100644 --- a/nixos/machines/orbi/social-matrix-terranix.nix +++ b/nixos/machines/orbi/social-matrix-terranix.nix @@ -68,7 +68,10 @@ in }; }; - config = { config, pkgs, lib, ... }: { + config = { config, lib, ... }: { + nixpkgs.pkgs = pkgs; + imports = [ ./container-monitoring.nix ]; + promtail.port = 3504; system.stateVersion = "23.11"; services.postgresql = {