From f328d740e05955edbee704437766546f3ac55b96 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Wed, 15 May 2024 17:11:41 +0200 Subject: [PATCH] make nginx logs properly work --- nixos/components/monitor/opentelemetry.nix | 39 ++++++++++----- nixos/components/network/nginx.nix | 33 +++++++++---- nixos/machines/chungus/configuration.nix | 1 - nixos/machines/chungus/telemetry/promtail.nix | 49 ------------------- nixos/system/all/nginx.nix | 42 +++++++--------- 5 files changed, 69 insertions(+), 95 deletions(-) delete mode 100644 nixos/machines/chungus/telemetry/promtail.nix diff --git a/nixos/components/monitor/opentelemetry.nix b/nixos/components/monitor/opentelemetry.nix index 0576a8d..0cc32ac 100644 --- a/nixos/components/monitor/opentelemetry.nix +++ b/nixos/components/monitor/opentelemetry.nix @@ -35,19 +35,32 @@ in # todo : make sure we filter out metrics from otlp receivers (mkIf config.components.monitor.enable { services.opentelemetry-collector.settings = { - service.pipelines.metrics.processors = [ "metricstransform" ]; - processors.metricstransform.transforms = [ - { - include = ".*"; - match_type = "regexp"; - action = "update"; - operations = [{ - action = "add_label"; - new_label = "machine"; - new_value = config.networking.hostName; - }]; - } - ]; + + service.pipelines.metrics.processors = [ "metricstransform" "resourcedetection/system" ]; + service.pipelines.logs.processors = [ "resourcedetection/system" ]; + + processors = { + + # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/resourcedetectionprocessor/README.md + "resourcedetection/system" = { + detectors = [ "system" ]; + override = false; + system.hostname_sources = [ "os" ]; + }; + + metricstransform.transforms = [ + { + include = ".*"; + match_type = "regexp"; + action = "update"; + operations = [{ + action = "add_label"; + new_label = "machine"; + new_value = config.networking.hostName; + }]; + } + ]; + }; }; }) diff --git a/nixos/components/network/nginx.nix b/nixos/components/network/nginx.nix index e43dc67..5823e1c 100644 --- a/nixos/components/network/nginx.nix +++ b/nixos/components/network/nginx.nix @@ -33,23 +33,38 @@ with lib; # for loki logging commonHttpConfig = '' - log_format logfmt escape=json 'timestamp=$time_iso8601 ' - 'facility=nginx ' - 'src_addr=$remote_addr ' - 'body_bytes_sent=$body_bytes_sent ' - 'request_time=$request_time ' - 'response_status=$status ' - 'request="$request" ' + log_format logfmt + 'timestamp="$time_iso8601" ' + 'facility="nginx" ' + 'src_addr="$remote_addr" ' + 'body_bytes_sent="$body_bytes_sent" ' + 'request_time="$request_time" ' + 'response_status="$status" ' 'request_method="$request_method" ' + 'request="$request" ' 'host="$host" ' 'upstream_cache_status="$upstream_cache_status" ' 'upstream_addr="$upstream_addr" ' 'http_x_forwarded_for="$http_x_forwarded_for" ' 'http_referrer="$http_referer" ' - 'http_user_agent="$http_user_agent"'; + 'http_user_agent="$http_user_agent" '; + + log_format json_combined escape=json + '{' + '"time_local":"$time_local",' + '"remote_addr":"$remote_addr",' + '"remote_user":"$remote_user",' + '"request":"$request",' + '"status": "$status",' + '"body_bytes_sent":"$body_bytes_sent",' + '"request_time":"$request_time",' + '"http_referrer":"$http_referer",' + '"http_user_agent":"$http_user_agent"' + '}'; + # log to local journald - access_log syslog:server=unix:/dev/log logfmt; + access_log syslog:server=unix:/dev/log,nohostname logfmt; ''; }; diff --git a/nixos/machines/chungus/configuration.nix b/nixos/machines/chungus/configuration.nix index 844c090..fb917a1 100644 --- a/nixos/machines/chungus/configuration.nix +++ b/nixos/machines/chungus/configuration.nix @@ -42,7 +42,6 @@ #./telemetry/opentelemetry-hass.nix ./telemetry/prometheus.nix ./telemetry/loki.nix - #./telemetry/promtail.nix #./home-display.nix diff --git a/nixos/machines/chungus/telemetry/promtail.nix b/nixos/machines/chungus/telemetry/promtail.nix deleted file mode 100644 index 2bce7e9..0000000 --- a/nixos/machines/chungus/telemetry/promtail.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ config, ... }: -{ - - services.opentelemetry-collector.settings = { - receivers.loki = { - protocols.http.endpoint = "127.0.0.1:3500"; - use_incoming_timestamp = true; - }; - service.pipelines.logs.receivers = [ "loki" ]; - }; - - services.promtail = { - enable = true; - configuration = { - server = { - http_listen_port = 28183; - grpc_listen_port = 0; - }; - - positions.filename = "/tmp/positions.yaml"; - clients = [ - { url = "http://127.0.0.1:3500/loki/api/v1/push"; } - ]; - - scrape_configs = [ - { - job_name = "journal"; - journal = { - max_age = "12h"; - #labels = { - # job = "systemd-journal"; - # host = config.networking.hostName; - #}; - }; - relabel_configs = [ - { - source_labels = [ "__journal__systemd_unit" ]; - target_label = "unit"; - } - { - source_labels = [ "__journal__transport" ]; - target_label = "transport"; - } - ]; - } - ]; - }; - }; -} diff --git a/nixos/system/all/nginx.nix b/nixos/system/all/nginx.nix index 3b5e4ac..996f24d 100644 --- a/nixos/system/all/nginx.nix +++ b/nixos/system/all/nginx.nix @@ -17,29 +17,25 @@ in recommendedTlsSettings = lib.mkDefault true; # for loki logging - commonHttpConfig = '' - log_format logfmt escape=json 'timestamp=$time_iso8601 ' - 'facility=nginx ' - 'src_addr=$remote_addr ' - 'body_bytes_sent=$body_bytes_sent ' - 'request_time=$request_time ' - 'response_status=$status ' - 'request="$request" ' - 'request_method="$request_method" ' - 'host="$host" ' - 'upstream_cache_status="$upstream_cache_status" ' - 'upstream_addr="$upstream_addr" ' - 'http_x_forwarded_for="$http_x_forwarded_for" ' - 'http_referrer="$http_referer" ' - 'http_user_agent="$http_user_agent"'; - - # log to local journald - access_log syslog:server=unix:/dev/log logfmt; - - # to send logs somewhere - #access_log syslog:server=${access_log_sink} logfmt; - #error_log syslog:server=${error_log_sink}; - ''; + #commonHttpConfig = '' + # log_format logfmt 'timestamp=$time_iso8601 ' + # 'facility=nginx ' + # 'src_addr=$remote_addr ' + # 'body_bytes_sent=$body_bytes_sent ' + # 'request_time=$request_time ' + # 'response_status=$status ' + # 'request="$request" ' + # 'request_method="$request_method" ' + # 'host="$host" ' + # 'upstream_cache_status="$upstream_cache_status" ' + # 'upstream_addr="$upstream_addr" ' + # 'http_x_forwarded_for="$http_x_forwarded_for" ' + # 'http_referrer="$http_referer" ' + # 'http_user_agent="$http_user_agent"'; +# +# # log to local journald +# access_log syslog:server=unix:/dev/log logfmt; +# ''; # for graylog logging