From 5fbe52eb1145eb3a559b6a31e962876a2b81aba7 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Wed, 15 May 2024 22:54:12 +0200 Subject: [PATCH] make loki, promtail better (better indices and so on) --- nixos/components/monitor/promtail.nix | 43 ++++++++++++++++++++--- nixos/components/network/nginx.nix | 1 - nixos/machines/chungus/media-curl.nix | 19 +++++----- nixos/machines/chungus/paperless.nix | 1 + nixos/machines/chungus/sync-rbackup.nix | 1 + nixos/machines/chungus/sync-syncoid.nix | 4 +++ nixos/machines/chungus/telemetry/loki.nix | 23 +++++++++++- nixos/system/all/nginx.nix | 8 ++--- 8 files changed, 81 insertions(+), 19 deletions(-) diff --git a/nixos/components/monitor/promtail.nix b/nixos/components/monitor/promtail.nix index 3b1a04f..f221dc8 100644 --- a/nixos/components/monitor/promtail.nix +++ b/nixos/components/monitor/promtail.nix @@ -46,16 +46,49 @@ in { job_name = "journal"; journal = { + json = true; max_age = "12h"; + labels.job = "systemd-journal"; }; - relabel_configs = [ + pipeline_stages = [ { - source_labels = [ "__journal__systemd_unit" ]; - target_label = "unit"; + json.expressions = { + transport = "_TRANSPORT"; + unit = "_SYSTEMD_UNIT"; + msg = "MESSAGE"; + coredump_cgroup = "COREDUMP_CGROUP"; + coredump_exe = "COREDUMP_EXE"; + coredump_cmdline = "COREDUMP_CMDLINE"; + coredump_uid = "COREDUMP_UID"; + coredump_gid = "COREDUMP_GID"; + }; } { - source_labels = [ "__journal__transport" ]; - target_label = "transport"; + # Set the unit (defaulting to the transport like audit and kernel) + template = { + source = "unit"; + template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}"; + }; + } + { labels.coredump_unit = "coredump_unit"; } + { + # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values + replace = { + source = "unit"; + expression = "^(session-\\d+.scope)$"; + replace = "session.scope"; + }; + } + { labels.unit = "unit"; } + { + # Write the proper message instead of JSON + output.source = "msg"; + } + ]; + relabel_configs = [ + { + source_labels = [ "__journal__hostname" ]; + target_label = "instance"; } ]; } diff --git a/nixos/components/network/nginx.nix b/nixos/components/network/nginx.nix index 5823e1c..6dfcf20 100644 --- a/nixos/components/network/nginx.nix +++ b/nixos/components/network/nginx.nix @@ -62,7 +62,6 @@ with lib; '"http_user_agent":"$http_user_agent"' '}'; - # log to local journald access_log syslog:server=unix:/dev/log,nohostname logfmt; ''; diff --git a/nixos/machines/chungus/media-curl.nix b/nixos/machines/chungus/media-curl.nix index 8eba567..9aa6722 100644 --- a/nixos/machines/chungus/media-curl.nix +++ b/nixos/machines/chungus/media-curl.nix @@ -2,19 +2,22 @@ with lib; let configuration = { - Chaospott37C3Tickets = rec { - url = "https://md.chaospott.de/171s8-_cQCyX_tUca_Jxqw/download"; - target = "/media/curl/37C3"; - options = [ - "-o $( date +%H:%M:%S )-TicketPlaning.md" - ]; - }; + + #Chaospott37C3Tickets = rec { + # url = "https://md.chaospott.de/171s8-_cQCyX_tUca_Jxqw/download"; + # target = "/media/curl/37C3"; + # options = [ + # "-o $( date +%H:%M:%S )-TicketPlaning.md" + # ]; + #}; + StableConfussion = { url = "http://stable-confusion.r/outputs/"; target = "/media/curl/stable-confusion"; - options = [ "--mirror" ]; + options = [ "--mirror" "--quiet" ]; command = "wget"; }; + }; downloadScript = diff --git a/nixos/machines/chungus/paperless.nix b/nixos/machines/chungus/paperless.nix index 0811d34..4f455d0 100644 --- a/nixos/machines/chungus/paperless.nix +++ b/nixos/machines/chungus/paperless.nix @@ -8,6 +8,7 @@ PAPERLESS_OCR_LANGUAGE = "deu+eng"; PAPERLESS_APP_TITLE = "paperless.chungus.private"; PAPERLESS_CONSUMER_IGNORE_PATTERN = builtins.toJSON [ ".DS_STORE/*" "desktop.ini" ]; + PAPERLESS_EMAIL_TASK_CRON = "0 */8 * * *"; # “At minute 0 past every 8th hour.” #PAPERLESS_CONSUMER_DELETE_DUPLICATES = true; }; }; diff --git a/nixos/machines/chungus/sync-rbackup.nix b/nixos/machines/chungus/sync-rbackup.nix index d492344..238f284 100644 --- a/nixos/machines/chungus/sync-rbackup.nix +++ b/nixos/machines/chungus/sync-rbackup.nix @@ -3,6 +3,7 @@ sops.secrets.rsync_private_key = { }; rbackup.plans = { + # todo : syncoid nextcloud = { sshKeyPath = config.sops.secrets.rsync_private_key.path; src = "root@orbi:/var/lib/nixos-containers/nextcloud"; diff --git a/nixos/machines/chungus/sync-syncoid.nix b/nixos/machines/chungus/sync-syncoid.nix index cce7f1b..0046017 100644 --- a/nixos/machines/chungus/sync-syncoid.nix +++ b/nixos/machines/chungus/sync-syncoid.nix @@ -8,6 +8,8 @@ services.syncoid = { enable = true; + + # local commands.service2 = { source = "zroot/services2"; target = "zraid/mirror/services2"; # should not be created up front! @@ -20,6 +22,8 @@ source = "zroot/postgresql"; target = "zraid/mirror/postgresql"; # should not be created up front! }; + + # remote commands.photoprism = { sshKey = config.sops.secrets.syncoid_private_key.path; source = "root@orbi:zmedia/photoprism"; diff --git a/nixos/machines/chungus/telemetry/loki.nix b/nixos/machines/chungus/telemetry/loki.nix index e230970..ca8b131 100644 --- a/nixos/machines/chungus/telemetry/loki.nix +++ b/nixos/machines/chungus/telemetry/loki.nix @@ -1,17 +1,38 @@ { config, pkgs, ... }: { - services.opentelemetry-collector.settings = { exporters.loki = { endpoint = "http://127.0.0.1:3100/loki/api/v1/push"; + default_labels_enabled = { + exporter = true; + job = true; + instance = true; + level = true; + }; + }; + processors = { + attributes.actions = [ + { + action = "insert"; + key = "loki.attribute.labels"; + value = "job, unit"; + } + ]; + resource.attributes = [{ + action = "insert"; + key = "loki.resource.labels"; + value = "host.name"; + }]; }; service.pipelines.logs.exporters = [ "loki" ]; + service.pipelines.logs.processors = [ "resource" "attributes" ]; }; services.loki = { enable = true; configuration = { + server = { http_listen_port = 3100; log_level = "warn"; diff --git a/nixos/system/all/nginx.nix b/nixos/system/all/nginx.nix index 996f24d..5add99e 100644 --- a/nixos/system/all/nginx.nix +++ b/nixos/system/all/nginx.nix @@ -32,10 +32,10 @@ in # 'http_x_forwarded_for="$http_x_forwarded_for" ' # 'http_referrer="$http_referer" ' # 'http_user_agent="$http_user_agent"'; -# -# # log to local journald -# access_log syslog:server=unix:/dev/log logfmt; -# ''; + # + # # log to local journald + # access_log syslog:server=unix:/dev/log logfmt; + # ''; # for graylog logging