add loki and grafana to robi

nixos/components/network/tinc/private.nix

@@ -15,9 +15,13 @@ let
     robi = "10.23.42.111";
   };
   subDomains = {
+    # robi
     "transmission.robi" = hosts.robi;
     "transmission2.robi" = hosts.robi;
     "loki.robi" = hosts.robi;
+    "grafana.robi" = hosts.robi;
+    "prometheus.robi" = hosts.robi;
+    # pepe
     "loki.pepe" = hosts.pepe;
     "grafana.pepe" = hosts.pepe;
     "prometheus.pepe" = hosts.pepe;

nixos/machines/pepe/configuration.nix

@@ -24,6 +24,7 @@
     ./wireguard.nix
     #./tts.nix
 
+    # logging
     ./loki.nix
     ./loki-promtail.nix
     ./prometheus.nix

nixos/machines/robi/configuration.nix

@@ -43,6 +43,12 @@
     #./weechat.nix
     #./property.nix # flask sucks, find something else
 
+    # logging
+    ./loki.nix
+    ./loki-promtail.nix
+    ./prometheus.nix
+    ./grafana.nix
+
   ];
 
   # 2 hours = 2 * 60 * 60 = 7200 seconds

nixos/machines/robi/gitea.nix

@@ -25,6 +25,7 @@
     settings = {
       service.DISABLE_REGISTRATION = true;
       session.COOKIE_SECURE = true;
+      log.LEVEL = "Warn";
       other = {
         SHOW_FOOTER_VERSION = false;
       };

nixos/machines/robi/grafana.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+{
+
+  services.nginx.virtualHosts.${config.services.grafana.domain} = {
+    extraConfig = ''
+      allow ${config.tinc.private.subnet};
+      deny all;
+    '';
+    locations."/" = {
+      proxyPass = "http://localhost:${toString config.services.grafana.port}";
+      proxyWebsockets = true;
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    domain = "grafana.robi.private";
+    port = 2342;
+    addr = "localhost";
+  };
+
+}

nixos/machines/robi/loki-promtail.nix

@@ -0,0 +1,35 @@
+{ config, ... }:
+{
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 28183;
+        grpc_listen_port = 0;
+      };
+      positions.filename = "/tmp/positions.yaml";
+      clients = [
+        { url = "http://127.0.0.1:3100/loki/api/v1/push"; }
+      ];
+
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName;
+            };
+          };
+          relabel_configs = [{
+            source_labels = [ "__journal__systemd_unit" ];
+            target_label = "unit";
+          }];
+        }
+      ];
+
+    };
+
+  };
+}

nixos/machines/robi/loki.nix

@@ -0,0 +1,99 @@
+{ config, pkgs, ... }:
+{
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3100;
+        log_level = "warn";
+      };
+      auth_enabled = false;
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-06-06";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      chunk_store_config = {
+        max_look_back_period = "0s";
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki";
+        shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+    };
+    # user, group, dataDir, extraFlags, (configFile)
+  };
+
+  #services.nginx = {
+  #  enable = true;
+  #  virtualHosts.loki = {
+  #    serverName = "loki.pepe.private";
+  #    locations."/" = {
+  #      proxyWebsockets = true;
+  #      proxyPass = "http://127.0.0.1:3100";
+  #      #extraConfig = ''
+  #      #  access_log off;
+  #      #  allow ${config.tinc.private.subnet};
+  #      #  deny all;
+  #      #'';
+  #    };
+  #  };
+  #};
+
+}

nixos/machines/robi/prometheus.nix

@@ -4,8 +4,12 @@
     enable = true;
     statusPage = true;
     virtualHosts = {
-      "prometheus.workhorse.private" = {
+      "prometheus.robi.private" = {
-        locations."/" = { proxyPass = "http://workhorse.private:9090"; };
+        extraConfig = ''
+          allow ${config.tinc.private.subnet};
+          deny all;
+        '';
+        locations."/" = { proxyPass = "http://localhost:${toString config.services.prometheus.port}"; };
       };
     };
   };
@@ -14,19 +18,16 @@
     enable = true;
     # keep data for 30 days
     extraFlags = [ "--storage.tsdb.retention.time=30d" ];
+
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+    };
+
     scrapeConfigs = [
-      {
-        job_name = "nginx";
-        scrape_interval = "8s";
-        static_configs = [
-          {
-            targets = [ "workhorse.private:9113" ];
-            labels = {
-              service = "nginx";
-            };
-          }
-        ];
-      }
       {
         job_name = "netdata";
         metrics_path = "/api/v1/allmetrics";
@@ -34,51 +35,20 @@
         scrape_interval = "5s";
         static_configs = [
           {
-            targets = [ "pepe.private:19999" ];
+            targets = [ "localhost:19999" ];
             labels = {
               service = "netdata";
-              server = "pepe";
+              server = "robi";
-            };
-          }
-          {
-            targets = [ "workhorse.private:19999" ];
-            labels = {
-              service = "netdata";
-              server = "workhorse";
             };
           }
         ];
       }
       {
-        job_name = "gogs";
+        job_name = "systemd";
-        metrics_path = "/-/metrics";
-        params.format = [ "prometheus" ];
-        scrape_interval = "10s";
         static_configs = [{
-          targets = [ "workhorse.private:3000" ];
+          targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
-          labels = {
-            service = "gogs";
-            server = "kruck";
-          };
         }];
       }
-      #{
-      #  job_name = "home-assistant";
-      #  scrape_interval = "60s";
-      #  metrics_path = "/api/prometheus";
-      #  # you can create this token on your user profile page
-      #  # http://pepe.private:8123/profile
-      #  bearer_token =
-      #    lib.fileContents <secrets/prometheus/home-assistant/api_token>;
-      #  static_configs = [{
-      #    targets = [ "pepe.private:8123" ];
-      #    labels = {
-      #      service = "hass";
-      #      server = "pepe";
-      #      city = "essen";
-      #    };
-      #  }];
-      #}
     ];
   };
 }

nixos/system/all/nginx-landingpage.nix

@@ -46,6 +46,22 @@
                 image = "https://media.giphy.com/media/fyLi0OuWysotq/giphy.gif";
               }
 
+              {
+                label = "robi - logs";
+                href = "http://grafana.robi.private/explore";
+                image = "https://raw.githubusercontent.com/cncf/landscape/master/hosted_logos/grafana-loki.svg";
+              }
+              {
+                label = "robi - grafana";
+                href = "http://grafana.robi.private/";
+                image = "https://www.vectorlogo.zone/logos/grafana/grafana-icon.svg";
+              }
+              {
+                label = "robi - prometheus";
+                href = "http://prometheus.robi.private/";
+                image = "https://www.vectorlogo.zone/logos/prometheusio/prometheusio-icon.svg";
+              }
+
               {
                 label = "pepe - logs";
                 href = "http://grafana.pepe.private/explore";
@@ -53,7 +69,7 @@
               }
               {
                 label = "pepe - grafana";
-                href = "http://http://grafana.pepe.private/";
+                href = "http://grafana.pepe.private/";
                 image = "https://www.vectorlogo.zone/logos/grafana/grafana-icon.svg";
               }
               {