make loki, promtail better

(better indices and so on)

nixos/components/monitor/promtail.nix

@@ -46,16 +46,49 @@ in
             {
               job_name = "journal";
               journal = {
+                json = true;
                 max_age = "12h";
+                labels.job = "systemd-journal";
               };
-              relabel_configs = [
+              pipeline_stages = [
                 {
-                  source_labels = [ "__journal__systemd_unit" ];
-                  target_label = "unit";
+                  json.expressions = {
+                    transport = "_TRANSPORT";
+                    unit = "_SYSTEMD_UNIT";
+                    msg = "MESSAGE";
+                    coredump_cgroup = "COREDUMP_CGROUP";
+                    coredump_exe = "COREDUMP_EXE";
+                    coredump_cmdline = "COREDUMP_CMDLINE";
+                    coredump_uid = "COREDUMP_UID";
+                    coredump_gid = "COREDUMP_GID";
+                  };
                 }
                 {
-                  source_labels = [ "__journal__transport" ];
-                  target_label = "transport";
+                  # Set the unit (defaulting to the transport like audit and kernel)
+                  template = {
+                    source = "unit";
+                    template = "{{if .unit}}{{.unit}}{{else}}{{.transport}}{{end}}";
+                  };
+                }
+                { labels.coredump_unit = "coredump_unit"; }
+                {
+                  # Normalize session IDs (session-1234.scope -> session.scope) to limit number of label values
+                  replace = {
+                    source = "unit";
+                    expression = "^(session-\\d+.scope)$";
+                    replace = "session.scope";
+                  };
+                }
+                { labels.unit = "unit"; }
+                {
+                  # Write the proper message instead of JSON
+                  output.source = "msg";
+                }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__journal__hostname" ];
+                  target_label = "instance";
                 }
               ];
             }

nixos/components/network/nginx.nix

@@ -62,7 +62,6 @@ with lib;
                 '"http_user_agent":"$http_user_agent"'
               '}';
 
-
           # log to local journald
           access_log syslog:server=unix:/dev/log,nohostname logfmt;
         '';

nixos/machines/chungus/media-curl.nix

@@ -2,19 +2,22 @@
 with lib;
 let
   configuration = {
-    Chaospott37C3Tickets = rec {
-      url = "https://md.chaospott.de/171s8-_cQCyX_tUca_Jxqw/download";
-      target = "/media/curl/37C3";
-      options = [
-        "-o $( date +%H:%M:%S )-TicketPlaning.md"
-      ];
-    };
+
+    #Chaospott37C3Tickets = rec {
+    #  url = "https://md.chaospott.de/171s8-_cQCyX_tUca_Jxqw/download";
+    #  target = "/media/curl/37C3";
+    #  options = [
+    #    "-o $( date +%H:%M:%S )-TicketPlaning.md"
+    #  ];
+    #};
+
     StableConfussion = {
       url = "http://stable-confusion.r/outputs/";
       target = "/media/curl/stable-confusion";
-      options = [ "--mirror" ];
+      options = [ "--mirror" "--quiet" ];
       command = "wget";
     };
+
   };
 
   downloadScript =

nixos/machines/chungus/paperless.nix

@@ -8,6 +8,7 @@
       PAPERLESS_OCR_LANGUAGE = "deu+eng";
       PAPERLESS_APP_TITLE = "paperless.chungus.private";
       PAPERLESS_CONSUMER_IGNORE_PATTERN = builtins.toJSON [ ".DS_STORE/*" "desktop.ini" ];
+      PAPERLESS_EMAIL_TASK_CRON = "0 */8 * * *"; # “At minute 0 past every 8th hour.”
       #PAPERLESS_CONSUMER_DELETE_DUPLICATES = true;
     };
   };

nixos/machines/chungus/sync-rbackup.nix

@@ -3,6 +3,7 @@
   sops.secrets.rsync_private_key = { };
 
   rbackup.plans = {
+    # todo : syncoid
     nextcloud = {
       sshKeyPath = config.sops.secrets.rsync_private_key.path;
       src = "root@orbi:/var/lib/nixos-containers/nextcloud";

nixos/machines/chungus/sync-syncoid.nix

@@ -8,6 +8,8 @@
 
   services.syncoid = {
     enable = true;
+
+    # local
     commands.service2 = {
       source = "zroot/services2";
       target = "zraid/mirror/services2"; # should not be created up front!
@@ -20,6 +22,8 @@
       source = "zroot/postgresql";
       target = "zraid/mirror/postgresql"; # should not be created up front!
     };
+
+    # remote
     commands.photoprism = {
       sshKey = config.sops.secrets.syncoid_private_key.path;
       source = "root@orbi:zmedia/photoprism";

nixos/machines/chungus/telemetry/loki.nix

@@ -1,17 +1,38 @@
 { config, pkgs, ... }:
 {
 
-
   services.opentelemetry-collector.settings = {
     exporters.loki = {
       endpoint = "http://127.0.0.1:3100/loki/api/v1/push";
+      default_labels_enabled = {
+        exporter = true;
+        job = true;
+        instance = true;
+        level = true;
+      };
+    };
+    processors = {
+      attributes.actions = [
+        {
+          action = "insert";
+          key = "loki.attribute.labels";
+          value = "job, unit";
+        }
+      ];
+      resource.attributes = [{
+        action = "insert";
+        key = "loki.resource.labels";
+        value = "host.name";
+      }];
     };
     service.pipelines.logs.exporters = [ "loki" ];
+    service.pipelines.logs.processors = [ "resource" "attributes" ];
   };
 
   services.loki = {
     enable = true;
     configuration = {
+
       server = {
         http_listen_port = 3100;
         log_level = "warn";

nixos/system/all/nginx.nix

@@ -32,10 +32,10 @@ in
     #    'http_x_forwarded_for="$http_x_forwarded_for" '
     #    'http_referrer="$http_referer" '
     #    'http_user_agent="$http_user_agent"';
-#
-#      # log to local journald
-#      access_log syslog:server=unix:/dev/log logfmt;
-#    '';
+    #
+    #      # log to local journald
+    #      access_log syslog:server=unix:/dev/log logfmt;
+    #    '';
 
 
     # for graylog logging