From 9160b3417841ae29f0a2323d6ae533f74f604c16 Mon Sep 17 00:00:00 2001
From: Ingolf Wagner <contact@ingolf-wagner.de>
Date: Thu, 16 May 2024 13:10:48 +0200
Subject: [PATCH] some improvements

---
 nixos/components/monitor/export-zfs.nix   |  2 +-
 nixos/components/monitor/promtail.nix     | 13 +++++++------
 nixos/machines/chungus/telemetry/loki.nix | 23 +++++++++++++----------
 3 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/nixos/components/monitor/export-zfs.nix b/nixos/components/monitor/export-zfs.nix
index 750e0e6..c416faf 100644
--- a/nixos/components/monitor/export-zfs.nix
+++ b/nixos/components/monitor/export-zfs.nix
@@ -10,7 +10,7 @@ with types;
   config = mkMerge [
     (mkIf config.components.monitor.exporters.zfs.enable {
 
-      services.telegraf.extraConfig.inputs.zfs = {};
+      services.telegraf.extraConfig.inputs.zfs = { };
 
       services.prometheus.exporters.zfs.enable = true;
       services.opentelemetry-collector.settings = {
diff --git a/nixos/components/monitor/promtail.nix b/nixos/components/monitor/promtail.nix
index f221dc8..47ce63c 100644
--- a/nixos/components/monitor/promtail.nix
+++ b/nixos/components/monitor/promtail.nix
@@ -52,10 +52,17 @@ in
               };
               pipeline_stages = [
                 {
+                  # journalctl -o json | jq and you'll see these
                   json.expressions = {
                     transport = "_TRANSPORT";
                     unit = "_SYSTEMD_UNIT";
                     msg = "MESSAGE";
+                    priority = "PRIORITY";
+                    facility = "SYSLOG_FACILITY";
+                    boot_id = "_BOOT_ID";
+                    instance = "_HOSTNAME";
+
+                    # coredump
                     coredump_cgroup = "COREDUMP_CGROUP";
                     coredump_exe = "COREDUMP_EXE";
                     coredump_cmdline = "COREDUMP_CMDLINE";
@@ -85,12 +92,6 @@ in
                   output.source = "msg";
                 }
               ];
-              relabel_configs = [
-                {
-                  source_labels = [ "__journal__hostname" ];
-                  target_label = "instance";
-                }
-              ];
             }
           ];
         };
diff --git a/nixos/machines/chungus/telemetry/loki.nix b/nixos/machines/chungus/telemetry/loki.nix
index 9890343..e87ac5e 100644
--- a/nixos/machines/chungus/telemetry/loki.nix
+++ b/nixos/machines/chungus/telemetry/loki.nix
@@ -1,10 +1,6 @@
 { config, pkgs, ... }:
 {
 
-
-
-
-
   services.opentelemetry-collector.settings = {
     exporters.loki = {
       endpoint = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
@@ -20,14 +16,21 @@
         {
           action = "insert";
           key = "loki.attribute.labels";
-          value = "job, unit";
+          value = "job, unit, boot_id, instance, facility, priority";
+        }
+      ];
+      resource.attributes = [
+        {
+          action = "insert";
+          key = "loki.resource.labels";
+          value = "host.name";
+        }
+        {
+          action = "insert";
+          key = "loki.format";
+          value = "raw";
         }
       ];
-      resource.attributes = [{
-        action = "insert";
-        key = "loki.resource.labels";
-        value = "host.name";
-      }];
     };
     service.pipelines.logs.exporters = [ "loki" ];
     service.pipelines.logs.processors = [ "resource" "attributes" ];