From 2fc339cdd30a8a6f84c933e3b2465107b728c50e Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Wed, 8 Feb 2023 00:19:09 +0100 Subject: [PATCH] add loki and grafana to robi --- nixos/components/network/tinc/private.nix | 4 + nixos/machines/pepe/configuration.nix | 1 + nixos/machines/robi/configuration.nix | 6 ++ nixos/machines/robi/gitea.nix | 1 + nixos/machines/robi/grafana.nix | 22 +++++ nixos/machines/robi/loki-promtail.nix | 35 ++++++++ nixos/machines/robi/loki.nix | 99 +++++++++++++++++++++++ nixos/machines/robi/prometheus.nix | 68 +++++----------- nixos/system/all/nginx-landingpage.nix | 18 ++++- 9 files changed, 204 insertions(+), 50 deletions(-) create mode 100644 nixos/machines/robi/grafana.nix create mode 100644 nixos/machines/robi/loki-promtail.nix create mode 100644 nixos/machines/robi/loki.nix diff --git a/nixos/components/network/tinc/private.nix b/nixos/components/network/tinc/private.nix index 9b2d057..5f490ca 100644 --- a/nixos/components/network/tinc/private.nix +++ b/nixos/components/network/tinc/private.nix @@ -15,9 +15,13 @@ let robi = "10.23.42.111"; }; subDomains = { + # robi "transmission.robi" = hosts.robi; "transmission2.robi" = hosts.robi; "loki.robi" = hosts.robi; + "grafana.robi" = hosts.robi; + "prometheus.robi" = hosts.robi; + # pepe "loki.pepe" = hosts.pepe; "grafana.pepe" = hosts.pepe; "prometheus.pepe" = hosts.pepe; diff --git a/nixos/machines/pepe/configuration.nix b/nixos/machines/pepe/configuration.nix index 86824ad..0bc0ea9 100644 --- a/nixos/machines/pepe/configuration.nix +++ b/nixos/machines/pepe/configuration.nix @@ -24,6 +24,7 @@ ./wireguard.nix #./tts.nix + # logging ./loki.nix ./loki-promtail.nix ./prometheus.nix diff --git a/nixos/machines/robi/configuration.nix b/nixos/machines/robi/configuration.nix index eb8b8f4..abd401c 100644 --- a/nixos/machines/robi/configuration.nix +++ b/nixos/machines/robi/configuration.nix @@ -43,6 +43,12 @@ #./weechat.nix #./property.nix # flask sucks, find something else + # logging + ./loki.nix + ./loki-promtail.nix + ./prometheus.nix + ./grafana.nix + ]; # 2 hours = 2 * 60 * 60 = 7200 seconds diff --git a/nixos/machines/robi/gitea.nix b/nixos/machines/robi/gitea.nix index ea98f39..7ce0ac6 100644 --- a/nixos/machines/robi/gitea.nix +++ b/nixos/machines/robi/gitea.nix @@ -25,6 +25,7 @@ settings = { service.DISABLE_REGISTRATION = true; session.COOKIE_SECURE = true; + log.LEVEL = "Warn"; other = { SHOW_FOOTER_VERSION = false; }; diff --git a/nixos/machines/robi/grafana.nix b/nixos/machines/robi/grafana.nix new file mode 100644 index 0000000..b84761a --- /dev/null +++ b/nixos/machines/robi/grafana.nix @@ -0,0 +1,22 @@ +{ config, ... }: +{ + + services.nginx.virtualHosts.${config.services.grafana.domain} = { + extraConfig = '' + allow ${config.tinc.private.subnet}; + deny all; + ''; + locations."/" = { + proxyPass = "http://localhost:${toString config.services.grafana.port}"; + proxyWebsockets = true; + }; + }; + + services.grafana = { + enable = true; + domain = "grafana.robi.private"; + port = 2342; + addr = "localhost"; + }; + +} diff --git a/nixos/machines/robi/loki-promtail.nix b/nixos/machines/robi/loki-promtail.nix new file mode 100644 index 0000000..56bb507 --- /dev/null +++ b/nixos/machines/robi/loki-promtail.nix @@ -0,0 +1,35 @@ +{ config, ... }: +{ + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 28183; + grpc_listen_port = 0; + }; + positions.filename = "/tmp/positions.yaml"; + clients = [ + { url = "http://127.0.0.1:3100/loki/api/v1/push"; } + ]; + + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + } + ]; + + }; + + }; +} diff --git a/nixos/machines/robi/loki.nix b/nixos/machines/robi/loki.nix new file mode 100644 index 0000000..c0eba2c --- /dev/null +++ b/nixos/machines/robi/loki.nix @@ -0,0 +1,99 @@ +{ config, pkgs, ... }: +{ + + services.loki = { + enable = true; + configuration = { + server = { + http_listen_port = 3100; + log_level = "warn"; + }; + auth_enabled = false; + + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 999999; + chunk_retain_period = "30s"; + max_transfer_retries = 0; + }; + + schema_config = { + configs = [{ + from = "2022-06-06"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + }]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + shared_store = "filesystem"; + }; + + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + + chunk_store_config = { + max_look_back_period = "0s"; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + + compactor = { + working_directory = "/var/lib/loki"; + shared_store = "filesystem"; + compactor_ring = { + kvstore = { + store = "inmemory"; + }; + }; + }; + }; + # user, group, dataDir, extraFlags, (configFile) + }; + + #services.nginx = { + # enable = true; + # virtualHosts.loki = { + # serverName = "loki.pepe.private"; + # locations."/" = { + # proxyWebsockets = true; + # proxyPass = "http://127.0.0.1:3100"; + # #extraConfig = '' + # # access_log off; + # # allow ${config.tinc.private.subnet}; + # # deny all; + # #''; + # }; + # }; + #}; + +} diff --git a/nixos/machines/robi/prometheus.nix b/nixos/machines/robi/prometheus.nix index 311c830..11da3a3 100644 --- a/nixos/machines/robi/prometheus.nix +++ b/nixos/machines/robi/prometheus.nix @@ -4,8 +4,12 @@ enable = true; statusPage = true; virtualHosts = { - "prometheus.workhorse.private" = { - locations."/" = { proxyPass = "http://workhorse.private:9090"; }; + "prometheus.robi.private" = { + extraConfig = '' + allow ${config.tinc.private.subnet}; + deny all; + ''; + locations."/" = { proxyPass = "http://localhost:${toString config.services.prometheus.port}"; }; }; }; }; @@ -14,19 +18,16 @@ enable = true; # keep data for 30 days extraFlags = [ "--storage.tsdb.retention.time=30d" ]; + + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9002; + }; + }; + scrapeConfigs = [ - { - job_name = "nginx"; - scrape_interval = "8s"; - static_configs = [ - { - targets = [ "workhorse.private:9113" ]; - labels = { - service = "nginx"; - }; - } - ]; - } { job_name = "netdata"; metrics_path = "/api/v1/allmetrics"; @@ -34,51 +35,20 @@ scrape_interval = "5s"; static_configs = [ { - targets = [ "pepe.private:19999" ]; + targets = [ "localhost:19999" ]; labels = { service = "netdata"; - server = "pepe"; - }; - } - { - targets = [ "workhorse.private:19999" ]; - labels = { - service = "netdata"; - server = "workhorse"; + server = "robi"; }; } ]; } { - job_name = "gogs"; - metrics_path = "/-/metrics"; - params.format = [ "prometheus" ]; - scrape_interval = "10s"; + job_name = "systemd"; static_configs = [{ - targets = [ "workhorse.private:3000" ]; - labels = { - service = "gogs"; - server = "kruck"; - }; + targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }]; } - #{ - # job_name = "home-assistant"; - # scrape_interval = "60s"; - # metrics_path = "/api/prometheus"; - # # you can create this token on your user profile page - # # http://pepe.private:8123/profile - # bearer_token = - # lib.fileContents ; - # static_configs = [{ - # targets = [ "pepe.private:8123" ]; - # labels = { - # service = "hass"; - # server = "pepe"; - # city = "essen"; - # }; - # }]; - #} ]; }; } diff --git a/nixos/system/all/nginx-landingpage.nix b/nixos/system/all/nginx-landingpage.nix index 1801bbb..a1f64dc 100644 --- a/nixos/system/all/nginx-landingpage.nix +++ b/nixos/system/all/nginx-landingpage.nix @@ -46,6 +46,22 @@ image = "https://media.giphy.com/media/fyLi0OuWysotq/giphy.gif"; } + { + label = "robi - logs"; + href = "http://grafana.robi.private/explore"; + image = "https://raw.githubusercontent.com/cncf/landscape/master/hosted_logos/grafana-loki.svg"; + } + { + label = "robi - grafana"; + href = "http://grafana.robi.private/"; + image = "https://www.vectorlogo.zone/logos/grafana/grafana-icon.svg"; + } + { + label = "robi - prometheus"; + href = "http://prometheus.robi.private/"; + image = "https://www.vectorlogo.zone/logos/prometheusio/prometheusio-icon.svg"; + } + { label = "pepe - logs"; href = "http://grafana.pepe.private/explore"; @@ -53,7 +69,7 @@ } { label = "pepe - grafana"; - href = "http://http://grafana.pepe.private/"; + href = "http://grafana.pepe.private/"; image = "https://www.vectorlogo.zone/logos/grafana/grafana-icon.svg"; } {