From e69fdc44d9f56a5e7e673ed80f485d0384f35934 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Wed, 15 May 2024 14:39:21 +0200 Subject: [PATCH] add promtail log shipping --- nixos/components/monitor/default.nix | 1 + nixos/components/monitor/opentelemetry.nix | 12 ++-- nixos/components/monitor/promtail.nix | 67 +++++++++++++++++++ nixos/machines/chungus/cache.nix | 3 +- nixos/machines/chungus/configuration.nix | 4 +- .../machines/chungus/{ => telemetry}/loki.nix | 8 +++ .../promtail.nix} | 22 ++++-- 7 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 nixos/components/monitor/promtail.nix rename nixos/machines/chungus/{ => telemetry}/loki.nix (91%) rename nixos/machines/chungus/{loki-promtail.nix => telemetry/promtail.nix} (63%) diff --git a/nixos/components/monitor/default.nix b/nixos/components/monitor/default.nix index bc91357..dc6bc1c 100644 --- a/nixos/components/monitor/default.nix +++ b/nixos/components/monitor/default.nix @@ -14,6 +14,7 @@ with types; ./netdata.nix ./opentelemetry.nix ./prometheus.nix + ./promtail.nix ./telegraf.nix ]; diff --git a/nixos/components/monitor/opentelemetry.nix b/nixos/components/monitor/opentelemetry.nix index f84107a..0576a8d 100644 --- a/nixos/components/monitor/opentelemetry.nix +++ b/nixos/components/monitor/opentelemetry.nix @@ -62,9 +62,9 @@ in pipelines.metrics = { exporters = [ "otlp" ]; }; - #pipelines.logs = { - # exporters = [ "otlp" ]; - #}; + pipelines.logs = { + exporters = [ "otlp" ]; + }; }; }; }) @@ -77,9 +77,9 @@ in pipelines.metrics = { receivers = [ "otlp" ]; }; - #pipelines.logs = { - # exporters = [ "otlp" ]; - #}; + pipelines.logs = { + receivers = [ "otlp" ]; + }; }; }; }) diff --git a/nixos/components/monitor/promtail.nix b/nixos/components/monitor/promtail.nix new file mode 100644 index 0000000..3b1a04f --- /dev/null +++ b/nixos/components/monitor/promtail.nix @@ -0,0 +1,67 @@ +{ config, lib, ... }: +with lib; +with types; +let + cfg = config.components.monitor.promtail; +in +{ + options.components.monitor.promtail = { + enable = mkOption { + type = lib.types.bool; + default = config.components.monitor.enable; + }; + port = mkOption { + type = int; + default = 3500; + description = "port to provide promtail export"; + }; + }; + + config = mkMerge [ + + (mkIf config.components.monitor.promtail.enable { + + services.opentelemetry-collector.settings = { + receivers.loki = { + protocols.http.endpoint = "127.0.0.1:${toString cfg.port}"; + use_incoming_timestamp = true; + }; + service.pipelines.logs.receivers = [ "loki" ]; + }; + + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + + positions.filename = "/tmp/positions.yaml"; + clients = [ + { url = "http://127.0.0.1:${toString cfg.port}/loki/api/v1/push"; } + ]; + + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + { + source_labels = [ "__journal__transport" ]; + target_label = "transport"; + } + ]; + } + ]; + }; + }; + }) + ]; +} diff --git a/nixos/machines/chungus/cache.nix b/nixos/machines/chungus/cache.nix index baa4944..76e2b00 100644 --- a/nixos/machines/chungus/cache.nix +++ b/nixos/machines/chungus/cache.nix @@ -1,7 +1,8 @@ { ... }: { + # port 5000 already in use. services.nix-serve = { - enable = true; + enable = false; # needed if i want to trust my own build packages and dirivations # nix-store --generate-binary-cache-key key-name secret-key-file public-key-file diff --git a/nixos/machines/chungus/configuration.nix b/nixos/machines/chungus/configuration.nix index 0a51cbb..844c090 100644 --- a/nixos/machines/chungus/configuration.nix +++ b/nixos/machines/chungus/configuration.nix @@ -35,14 +35,14 @@ ./media-syncthing.nix # logging - ./loki.nix - ./loki-promtail.nix ./grafana.nix ./telemetry/telegraf-smart.nix ./telemetry/telegraf.nix #./telemetry/opentelemetry-hass.nix ./telemetry/prometheus.nix + ./telemetry/loki.nix + #./telemetry/promtail.nix #./home-display.nix diff --git a/nixos/machines/chungus/loki.nix b/nixos/machines/chungus/telemetry/loki.nix similarity index 91% rename from nixos/machines/chungus/loki.nix rename to nixos/machines/chungus/telemetry/loki.nix index c0eba2c..e230970 100644 --- a/nixos/machines/chungus/loki.nix +++ b/nixos/machines/chungus/telemetry/loki.nix @@ -1,6 +1,14 @@ { config, pkgs, ... }: { + + services.opentelemetry-collector.settings = { + exporters.loki = { + endpoint = "http://127.0.0.1:3100/loki/api/v1/push"; + }; + service.pipelines.logs.exporters = [ "loki" ]; + }; + services.loki = { enable = true; configuration = { diff --git a/nixos/machines/chungus/loki-promtail.nix b/nixos/machines/chungus/telemetry/promtail.nix similarity index 63% rename from nixos/machines/chungus/loki-promtail.nix rename to nixos/machines/chungus/telemetry/promtail.nix index 1c64287..2bce7e9 100644 --- a/nixos/machines/chungus/loki-promtail.nix +++ b/nixos/machines/chungus/telemetry/promtail.nix @@ -1,5 +1,14 @@ { config, ... }: { + + services.opentelemetry-collector.settings = { + receivers.loki = { + protocols.http.endpoint = "127.0.0.1:3500"; + use_incoming_timestamp = true; + }; + service.pipelines.logs.receivers = [ "loki" ]; + }; + services.promtail = { enable = true; configuration = { @@ -7,9 +16,10 @@ http_listen_port = 28183; grpc_listen_port = 0; }; + positions.filename = "/tmp/positions.yaml"; clients = [ - { url = "http://127.0.0.1:3100/loki/api/v1/push"; } + { url = "http://127.0.0.1:3500/loki/api/v1/push"; } ]; scrape_configs = [ @@ -17,10 +27,10 @@ job_name = "journal"; journal = { max_age = "12h"; - labels = { - job = "systemd-journal"; - host = config.networking.hostName; - }; + #labels = { + # job = "systemd-journal"; + # host = config.networking.hostName; + #}; }; relabel_configs = [ { @@ -34,8 +44,6 @@ ]; } ]; - }; - }; }