nixos-config/terranix/graylog/nginx.nix
Ingolf Wagner 973e3b3305
reformat
2021-11-01 19:30:41 +01:00

311 lines
9.6 KiB
Nix

/* # use this nginx configuration
# to send data to these inputs
log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
'"facility": "nginx", '
'"src_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent" }';
access_log syslog:server=${access_log_input} graylog2_json;
error_log syslog:server=${error_log_input};
*/
with builtins; {
resource = {
graylog_input = {
nginx_access_logs = {
title = "nginx access log";
# https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
global = true;
attributes = toJSON ({
allow_override_date = true;
bind_address = "0.0.0.0";
expand_structured_data = false;
force_rdns = false;
number_worker_threads = 4;
port = 12304;
recv_buffer_size = 1048576;
store_full_message = false;
});
};
nginx_error_logs = {
title = "nginx error log";
# https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
global = true;
attributes = toJSON ({
allow_override_date = true;
bind_address = "0.0.0.0";
expand_structured_data = false;
force_rdns = false;
number_worker_threads = 4;
port = 12305;
recv_buffer_size = 1048576;
store_full_message = false;
});
};
};
graylog_extractor = {
# nginx error
nginx_error_timestamp = {
input_id = "\${graylog_input.nginx_error_logs.id}";
order = 0;
title = "Timestamp";
type = "regex";
extractor_config = toJSON ({
regex_value =
"^.*:\\s(\\d\\d\\d\\d/\\d\\d/\\d\\d\\s\\d\\d:\\d\\d:\\d\\d)\\s.*$";
});
target_field = "timestamp";
source_field = "message";
cursor_strategy = "copy";
condition_type = "none";
converters = {
config = toJSON ({ date_format = "yyyy/MM/dd HH:mm:ss "; });
type = "date";
};
};
nginx_error_server = {
input_id = "\${graylog_input.nginx_error_logs.id}";
type = "regex";
source_field = "message";
cursor_strategy = "copy";
condition_type = "string";
condition_value = "server";
extractor_config = toJSON ({ regex_value = "server:\\s(.+?)(,|$)"; });
order = 1;
target_field = "server";
title = "server";
};
nginx_error_remote_addr = {
input_id = "\${graylog_input.nginx_error_logs.id}";
type = "regex";
source_field = "message";
cursor_strategy = "copy";
condition_type = "string";
condition_value = "client";
extractor_config = toJSON ({ regex_value = "client:\\s(.+?)(,|$)"; });
order = 2;
target_field = "remote_addr";
title = "remote_addr/client";
};
nginx_error_host = {
input_id = "\${graylog_input.nginx_error_logs.id}";
type = "regex";
source_field = "message";
cursor_strategy = "copy";
condition_type = "string";
condition_value = "host";
extractor_config = toJSON ({ regex_value = ''host:\s"(.+?)"(,|$)''; });
order = 3;
target_field = "host";
title = "host";
};
nginx_error_request_path = {
input_id = "\${graylog_input.nginx_error_logs.id}";
type = "regex";
source_field = "message";
cursor_strategy = "copy";
condition_type = "string";
condition_value = "request";
extractor_config =
toJSON ({ regex_value = ''request:\s"(.+?)"(,|$)''; });
order = 4;
target_field = "request_path";
title = "request_path/request";
};
nginx_error_request_verb = {
input_id = "\${graylog_input.nginx_error_logs.id}";
type = "regex";
source_field = "message";
cursor_strategy = "copy";
condition_type = "string";
condition_value = "request";
extractor_config = toJSON ({
regex_value = ''
request:\s"(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT|PATCH).+"(,|$)'';
});
order = 5;
target_field = "request_verb";
title = "request_verb";
};
# nginx access
nginx_access_json_from_syslog = {
input_id = "\${graylog_input.nginx_access_logs.id}";
title = "Get JSON from syslog message";
type = "regex";
cursor_strategy = "copy";
condition_type = "none";
source_field = "message";
target_field = "json";
order = 0;
extractor_config = toJSON ({ regex_value = "nginx:\\s+(.*)"; });
};
nginx_access_extract_json = {
input_id = "\${graylog_input.nginx_access_logs.id}";
title = "Extract JSON fields";
order = 1;
source_field = "json";
type = "json";
cursor_strategy = "copy";
condition_type = "none";
extractor_config = toJSON ({
flatten = true;
list_separator = ", ";
kv_separator = "=";
key_prefix = "";
key_separator = "_";
replace_key_whitespace = false;
key_whitespace_replacement = "_";
});
};
nginx_access_empty_json = {
input_id = "\${graylog_input.nginx_access_logs.id}";
order = 2;
title = "Empty JSON field";
type = "regex_replace";
extractor_config = toJSON ({
regex = ".*";
replacement = "-";
});
target_field = "json";
source_field = "json";
cursor_strategy = "copy";
condition_type = "none";
};
nginx_access_reduce_message = {
input_id = "\${graylog_input.nginx_access_logs.id}";
order = 3;
title = "Reduced message to path";
type = "regex_replace";
extractor_config = toJSON ({
regex = ''.*request": "(.*?)".*'';
replacement = "$1";
});
target_field = "message";
source_field = "message";
cursor_strategy = "copy";
condition_type = "none";
};
};
graylog_input_static_fields = {
nginx_access_logs = {
input_id = "\${graylog_input.nginx_access_logs.id}";
fields = {
from_nginx = true;
nginx_error = false;
nginx_access = true;
};
};
nginx_error_logs = {
input_id = "\${graylog_input.nginx_error_logs.id}";
fields = {
from_nginx = true;
nginx_error = true;
nginx_access = false;
};
};
};
graylog_stream = {
nginx5xx = {
title = "nginx 5xx";
description = "all requests answered with a 5xx response";
index_set_id = "\${graylog_index_set.default.id}";
disabled = false;
matching_type = "AND";
};
nginx4xx = {
title = "nginx 4xx";
description = "all requests answered with a 4xx response";
index_set_id = "\${graylog_index_set.default.id}";
disabled = false;
matching_type = "AND";
};
nginx2xx = {
title = "nginx 2xx";
description = "all requests answered with a 2xx response";
index_set_id = "\${graylog_index_set.default.id}";
disabled = false;
matching_type = "AND";
};
nginx_access = {
title = "nginx access";
description = "all requests";
index_set_id = "\${graylog_index_set.default.id}";
disabled = false;
matching_type = "AND";
};
nginx_error = {
title = "nginx error";
description = "all errors";
index_set_id = "\${graylog_index_set.default.id}";
disabled = false;
matching_type = "AND";
};
};
graylog_stream_rule =
let
nq_stream_rule = field: value: stream_id: {
inherit field value stream_id;
type = 1;
inverted = true;
};
eq_stream_rule = field: value: stream_id: {
inherit field value stream_id;
type = 1;
inverted = false;
};
gt_stream_rule = field: value: stream_id: {
inherit field value stream_id;
type = 3;
inverted = false;
};
lt_stream_rule = field: value: stream_id: {
inherit field value stream_id;
type = 4;
inverted = false;
};
between = min: max: stream_id: {
"is_nginx_access_${min}_${max}" =
(eq_stream_rule "nginx_access" true stream_id);
"nginx_above${min}" = (gt_stream_rule "response_status" min stream_id);
"nginx_below${max}" = (lt_stream_rule "response_status" max stream_id);
};
in
(between "499" "600" "\${graylog_stream.nginx5xx.id}")
// (between "399" "500" "\${graylog_stream.nginx4xx.id}")
// (between "199" "300" "\${graylog_stream.nginx2xx.id}") // {
is_nginx_access = (eq_stream_rule "nginx_access" true
"\${graylog_stream.nginx_access.id}");
is_nginx_error =
(eq_stream_rule "nginx_error" true "\${graylog_stream.nginx_error.id}");
};
};
}