graylog: improve alerting

configs/workhorse/graylog.nix

@@ -100,4 +100,27 @@ in {
     '';
   };
 
+  environment.etc."graylog/server/known_servers.csv" = {
+    enable = true;
+    text = ''
+      "ip","host_name"
+      "95.216.1.150","lassul.us"
+    '';
+  };
+
+  environment.etc."graylog/systemd/loglevel.csv" = {
+    enable = true;
+    text = ''
+      "value","Servity","Description"
+      "0","emergency","System is unusable"
+      "1","alert","Should be corrected immediately"
+      "2","cirtical","Critical conditions"
+      "3","error","Error Condition"
+      "4","warning","May indicate that an error will occur if action is not taken."
+      "5","notice","Events that are unusual, but not error conditions."
+      "6","info","Normal operational messages that require no action."
+      "7","debug","Information useful to developers for debugging the application."
+    '';
+  };
+
 }

configs/workhorse/mail-fetcher.nix

@@ -279,7 +279,7 @@
       "${config.users.users.mailUser.home}/.config/notmuch/notmuchrc";
     script = ''
       echo "run afew"
-      ${pkgs.afew}/bin/afew --all --tag -v
+      ${pkgs.afew}/bin/afew --all --tag
     '';
   };
   systemd.timers.retagmail = {

configs/workhorse/mattermost.nix

@@ -54,7 +54,7 @@ in {
         in ''
           log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
             '"facility": "nginx", '
-            '"remote_addr": "$remote_addr", '
+            '"src_addr": "$remote_addr", '
             '"body_bytes_sent": $body_bytes_sent, '
             '"request_time": $request_time, '
             '"response_status": $status, '

configs/workhorse/nextcloud.nix

@@ -71,7 +71,7 @@ in {
         in ''
           log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
             '"facility": "nginx", '
-            '"remote_addr": "$remote_addr", '
+            '"src_addr": "$remote_addr", '
             '"body_bytes_sent": $body_bytes_sent, '
             '"request_time": $request_time, '
             '"response_status": $status, '

modules/system/on-failure.nix

@@ -51,7 +51,7 @@ let
       --include \
       --request POST \
       --data-urlencode \
-      'payload={"text": "Service Failed ${plan.name}"}' \
+      'payload={"text": "Service Failed ${plan.name} on ${config.networking.hostName}"}' \
       ${cfg.url}
     '';
 

system/all/nginx.nix

@@ -16,7 +16,7 @@ in {
     commonHttpConfig = ''
       log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
         '"facility": "nginx", '
-        '"remote_addr": "$remote_addr", '
+        '"src_addr": "$remote_addr", '
         '"body_bytes_sent": $body_bytes_sent, '
         '"request_time": $request_time, '
         '"response_status": $status, '

system/all/packages.nix

@@ -41,7 +41,6 @@ let
       EOF
     '';
 
-
 in {
 
   programs.tmux = {