From 3a4ed070f26735f6a18020d72610049bffdb9ec3 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Sat, 18 May 2024 01:04:39 +0200 Subject: [PATCH] Properly handle labels in promtail --- nixos/components/monitor/promtail.nix | 181 ++++++++++++++++------ nixos/machines/chungus/telemetry/loki.nix | 2 +- 2 files changed, 131 insertions(+), 52 deletions(-) diff --git a/nixos/components/monitor/promtail.nix b/nixos/components/monitor/promtail.nix index 264755e..986e546 100644 --- a/nixos/components/monitor/promtail.nix +++ b/nixos/components/monitor/promtail.nix @@ -42,58 +42,137 @@ in { url = "http://127.0.0.1:${toString cfg.port}/loki/api/v1/push"; } ]; - scrape_configs = [ - { - job_name = "journal"; - journal = { - json = true; - max_age = "12h"; - labels.job = "systemd-journal"; - }; - pipeline_stages = [ - { - # journalctl -o json | jq and you'll see these - json.expressions = { - transport = "_TRANSPORT"; - unit = "_SYSTEMD_UNIT"; - msg = "MESSAGE"; - priority = "PRIORITY"; - facility = "SYSLOG_FACILITY"; - boot_id = "_BOOT_ID"; - instance = "_HOSTNAME"; + scrape_configs = - # coredump - coredump_cgroup = "COREDUMP_CGROUP"; - coredump_exe = "COREDUMP_EXE"; - coredump_cmdline = "COREDUMP_CMDLINE"; - coredump_uid = "COREDUMP_UID"; - coredump_gid = "COREDUMP_GID"; - }; - } - { - # Set the unit (defaulting to the transport like audit and kernel) - template = { - source = "unit"; - template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}"; - }; - } - { labels.coredump_unit = "coredump_unit"; } - { - # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values - replace = { - source = "unit"; - expression = "^(session-\\d+.scope)$"; - replace = "session.scope"; - }; - } - { labels.unit = "unit"; } - { - # Write the proper message instead of JSON - output.source = "msg"; - } - ]; - } - ]; + let + _replace = index: replacement: ''{{ Replace .Value "${toString index}" "${replacement}" 1 }}''; + _elseif = index: ''{{ else if eq .Value "${toString index}" }}''; + _if = index: ''{{ if eq .Value "${toString index}" }}''; + _end = ''{{ end }}''; + elseblock = index: replacement: "${_elseif index}${_replace index replacement}"; + ifblock = index: replacement: "${_if index}${_replace index replacement}"; + createTemplateLine = list: "${concatStrings (imap0 (index: replacement: if index == 0 then ifblock index replacement else elseblock index replacement) list)}${_end}"; + in + [ + { + job_name = "journal"; + journal = { + json = true; + max_age = "12h"; + labels.job = "systemd-journal"; + }; + pipeline_stages = [ + { + # Set of key/value pairs of JMESPath expressions. The key will be + # the key in the extracted data while the expression will be the value, + # evaluated as a JMESPath from the source data. + json.expressions = { + # journalctl -o json | jq and you'll see these + boot_id = "_BOOT_ID"; + facility = "SYSLOG_FACILITY"; + facility_label = "SYSLOG_FACILITY"; + instance = "_HOSTNAME"; + msg = "MESSAGE"; + priority = "PRIORITY"; + priority_label = "PRIORITY"; + transport = "_TRANSPORT"; + unit = "_SYSTEMD_UNIT"; + # coredump + #coredump_cgroup = "COREDUMP_CGROUP"; + #coredump_exe = "COREDUMP_EXE"; + #coredump_cmdline = "COREDUMP_CMDLINE"; + #coredump_uid = "COREDUMP_UID"; + #coredump_gid = "COREDUMP_GID"; + }; + } + { + # Set the unit (defaulting to the transport like audit and kernel) + template = { + source = "unit"; + template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}"; + }; + } + { + # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values + replace = { + source = "unit"; + expression = "^(session-\\d+.scope)$"; + replace = "session.scope"; + }; + } + { + # Map priority to human readable + template = { + source = "priority_label"; + #template = ''{{ if eq .Value "0" }}{{ Replace .Value "0" "emerg" 1 }}{{ else if eq .Value "1" }}{{ Replace .Value "1" "alert" 1 }}{{ else if eq .Value "2" }}{{ Replace .Value "2" "crit" 1 }}{{ else if eq .Value "3" }}{{ Replace .Value "3" "err" 1 }}{{ else if eq .Value "4" }}{{ Replace .Value "4" "warning" 1 }}{{ else if eq .Value "5" }}{{ Replace .Value "5" "notice" 1 }}{{ else if eq .Value "6" }}{{ Replace .Value "6" "info" 1 }}{{ else if eq .Value "7" }}{{ Replace .Value "7" "debug" 1 }}{{ end }}''; + template = createTemplateLine [ + "emergency" + "alert" + "critical" + "error" + "warning" + "notice" + "info" + "debug" + ]; + }; + } + { + # Map facility to human readable + template = + { + source = "facility_label"; + template = createTemplateLine [ + "kern" # Kernel messages + "user" # User-level messages + "mail" # Mail system Archaic POSIX still supported and sometimes used (for more mail(1)) + "daemon" # System daemons All daemons, including systemd and its subsystems + "auth" # Security/authorization messages Also watch for different facility 10 + "syslog" # Messages generated internally by syslogd For syslogd implementations (not used by systemd, see facility 3) + "lpr" # Line printer subsystem (archaic subsystem) + "news" # Network news subsystem (archaic subsystem) + "uucp" # UUCP subsystem (archaic subsystem) + "clock" # Clock daemon systemd-timesyncd + "authpriv" # Security/authorization messages Also watch for different facility 4 + "ftp" # FTP daemon + "-" # NTP subsystem + "-" # Log audit + "-" # Log alert + "cron" # Scheduling daemon + "local0" # Local use 0 (local0) + "local1" # Local use 1 (local1) + "local2" # Local use 2 (local2) + "local3" # Local use 3 (local3) + "local4" # Local use 4 (local4) + "local5" # Local use 5 (local5) + "local6" # Local use 6 (local6) + "local7" # Local use 7 (local7) + ]; + }; + } + { + # Key is REQUIRED and the name for the label that will be created. + # Value is optional and will be the name from extracted data whose value + # will be used for the value of the label. If empty, the value will be + # inferred to be the same as the key. + labels = { + boot_id = ""; + facility = ""; + facility_label = ""; + instance = ""; + priority = ""; + priority_label = ""; + transport = ""; + unit = ""; + }; + } + { + # Write the proper message instead of JSON + output.source = "msg"; + } + ]; + } + ]; }; }; }) diff --git a/nixos/machines/chungus/telemetry/loki.nix b/nixos/machines/chungus/telemetry/loki.nix index c934bcd..da030b3 100644 --- a/nixos/machines/chungus/telemetry/loki.nix +++ b/nixos/machines/chungus/telemetry/loki.nix @@ -16,7 +16,7 @@ { action = "insert"; key = "loki.attribute.labels"; - value = "job, unit, boot_id, instance, facility, priority"; + value = "job, unit, boot_id, instance, facility, facility_label, priority, priority_label"; } ]; resource.attributes = [