palo/nixos-config
palo/nixos-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Properly handle labels in promtail

Browse source
This commit is contained in:
Ingolf Wagner 2024-05-18 01:04:39 +02:00
parent 09138dc3a4
commit 3a4ed070f2
No known key found for this signature in database
GPG key ID: 76BF5F1928B9618B
2 changed files with 131 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
nixos/components/monitor/promtail.nix
View file

@ -42,7 +42,18 @@ in
{ url = "http://127.0.0.1:${toString cfg.port}/loki/api/v1/push"; } { url = "http://127.0.0.1:${toString cfg.port}/loki/api/v1/push"; }
]; ];
scrape_configs = [ scrape_configs =
let
_replace = index: replacement: ''{{ Replace .Value "${toString index}" "${replacement}" 1 }}'';
_elseif = index: ''{{ else if eq .Value "${toString index}" }}'';
_if = index: ''{{ if eq .Value "${toString index}" }}'';
_end = ''{{ end }}'';
elseblock = index: replacement: "${_elseif index}${_replace index replacement}";
ifblock = index: replacement: "${_if index}${_replace index replacement}";
createTemplateLine = list: "${concatStrings (imap0 (index: replacement: if index == 0 then ifblock index replacement else elseblock index replacement) list)}${_end}";
in
[
{ {
job_name = "journal"; job_name = "journal";
journal = { journal = {
@ -52,22 +63,26 @@ in
}; };
pipeline_stages = [ pipeline_stages = [
{ {
# journalctl -o json | jq and you'll see these # Set of key/value pairs of JMESPath expressions. The key will be
# the key in the extracted data while the expression will be the value,
# evaluated as a JMESPath from the source data.
json.expressions = { json.expressions = {
transport = "_TRANSPORT"; # journalctl -o json | jq and you'll see these
unit = "_SYSTEMD_UNIT"; boot_id = "_BOOT_ID";
facility = "SYSLOG_FACILITY";
facility_label = "SYSLOG_FACILITY";
instance = "_HOSTNAME";
msg = "MESSAGE"; msg = "MESSAGE";
priority = "PRIORITY"; priority = "PRIORITY";
facility = "SYSLOG_FACILITY"; priority_label = "PRIORITY";
boot_id = "_BOOT_ID"; transport = "_TRANSPORT";
instance = "_HOSTNAME"; unit = "_SYSTEMD_UNIT";
# coredump # coredump
coredump_cgroup = "COREDUMP_CGROUP"; #coredump_cgroup = "COREDUMP_CGROUP";
coredump_exe = "COREDUMP_EXE"; #coredump_exe = "COREDUMP_EXE";
coredump_cmdline = "COREDUMP_CMDLINE"; #coredump_cmdline = "COREDUMP_CMDLINE";
coredump_uid = "COREDUMP_UID"; #coredump_uid = "COREDUMP_UID";
coredump_gid = "COREDUMP_GID"; #coredump_gid = "COREDUMP_GID";
}; };
} }
{ {
@ -77,7 +92,6 @@ in
template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}"; template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
}; };
} }
{ labels.coredump_unit = "coredump_unit"; }
{ {
# Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
replace = { replace = {
@ -86,7 +100,72 @@ in
replace = "session.scope"; replace = "session.scope";
}; };
} }
{ labels.unit = "unit"; } {
# Map priority to human readable
template = {
source = "priority_label";
#template = ''{{ if eq .Value "0" }}{{ Replace .Value "0" "emerg" 1 }}{{ else if eq .Value "1" }}{{ Replace .Value "1" "alert" 1 }}{{ else if eq .Value "2" }}{{ Replace .Value "2" "crit" 1 }}{{ else if eq .Value "3" }}{{ Replace .Value "3" "err" 1 }}{{ else if eq .Value "4" }}{{ Replace .Value "4" "warning" 1 }}{{ else if eq .Value "5" }}{{ Replace .Value "5" "notice" 1 }}{{ else if eq .Value "6" }}{{ Replace .Value "6" "info" 1 }}{{ else if eq .Value "7" }}{{ Replace .Value "7" "debug" 1 }}{{ end }}'';
template = createTemplateLine [
"emergency"
"alert"
"critical"
"error"
"warning"
"notice"
"info"
"debug"
];
};
}
{
# Map facility to human readable
template =
{
source = "facility_label";
template = createTemplateLine [
"kern" # Kernel messages
"user" # User-level messages
"mail" # Mail system Archaic POSIX still supported and sometimes used (for more mail(1))
"daemon" # System daemons All daemons, including systemd and its subsystems
"auth" # Security/authorization messages Also watch for different facility 10
"syslog" # Messages generated internally by syslogd For syslogd implementations (not used by systemd, see facility 3)
"lpr" # Line printer subsystem (archaic subsystem)
"news" # Network news subsystem (archaic subsystem)
"uucp" # UUCP subsystem (archaic subsystem)
"clock" # Clock daemon systemd-timesyncd
"authpriv" # Security/authorization messages Also watch for different facility 4
"ftp" # FTP daemon
"-" # NTP subsystem
"-" # Log audit
"-" # Log alert
"cron" # Scheduling daemon
"local0" # Local use 0 (local0)
"local1" # Local use 1 (local1)
"local2" # Local use 2 (local2)
"local3" # Local use 3 (local3)
"local4" # Local use 4 (local4)
"local5" # Local use 5 (local5)
"local6" # Local use 6 (local6)
"local7" # Local use 7 (local7)
];
};
}
{
# Key is REQUIRED and the name for the label that will be created.
# Value is optional and will be the name from extracted data whose value
# will be used for the value of the label. If empty, the value will be
# inferred to be the same as the key.
labels = {
boot_id = "";
facility = "";
facility_label = "";
instance = "";
priority = "";
priority_label = "";
transport = "";
unit = "";
};
}
{ {
# Write the proper message instead of JSON # Write the proper message instead of JSON
output.source = "msg"; output.source = "msg";

2
nixos/machines/chungus/telemetry/loki.nix
View file

@ -16,7 +16,7 @@
{ {
action = "insert"; action = "insert";
key = "loki.attribute.labels"; key = "loki.attribute.labels";
value = "job, unit, boot_id, instance, facility, priority"; value = "job, unit, boot_id, instance, facility, facility_label, priority, priority_label";
} }
]; ];
resource.attributes = [ resource.attributes = [