311 lines
9.6 KiB
Nix
311 lines
9.6 KiB
Nix
/* # use this nginx configuration
|
|
# to send data to these inputs
|
|
|
|
log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
|
|
'"facility": "nginx", '
|
|
'"src_addr": "$remote_addr", '
|
|
'"body_bytes_sent": $body_bytes_sent, '
|
|
'"request_time": $request_time, '
|
|
'"response_status": $status, '
|
|
'"request": "$request", '
|
|
'"request_method": "$request_method", '
|
|
'"host": "$host",'
|
|
'"upstream_cache_status": "$upstream_cache_status",'
|
|
'"upstream_addr": "$upstream_addr",'
|
|
'"http_x_forwarded_for": "$http_x_forwarded_for",'
|
|
'"http_referrer": "$http_referer", '
|
|
'"http_user_agent": "$http_user_agent" }';
|
|
|
|
access_log syslog:server=${access_log_input} graylog2_json;
|
|
error_log syslog:server=${error_log_input};
|
|
*/
|
|
|
|
with builtins; {
|
|
|
|
resource = {
|
|
|
|
graylog_input = {
|
|
|
|
nginx_access_logs = {
|
|
title = "nginx access log";
|
|
# https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
|
|
type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
|
|
global = true;
|
|
attributes = toJSON ({
|
|
allow_override_date = true;
|
|
bind_address = "0.0.0.0";
|
|
expand_structured_data = false;
|
|
force_rdns = false;
|
|
number_worker_threads = 4;
|
|
port = 12304;
|
|
recv_buffer_size = 1048576;
|
|
store_full_message = false;
|
|
});
|
|
};
|
|
|
|
nginx_error_logs = {
|
|
title = "nginx error log";
|
|
# https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
|
|
type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
|
|
global = true;
|
|
attributes = toJSON ({
|
|
allow_override_date = true;
|
|
bind_address = "0.0.0.0";
|
|
expand_structured_data = false;
|
|
force_rdns = false;
|
|
number_worker_threads = 4;
|
|
port = 12305;
|
|
recv_buffer_size = 1048576;
|
|
store_full_message = false;
|
|
});
|
|
};
|
|
|
|
};
|
|
|
|
graylog_extractor = {
|
|
|
|
# nginx error
|
|
nginx_error_timestamp = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
order = 0;
|
|
title = "Timestamp";
|
|
type = "regex";
|
|
extractor_config = toJSON ({
|
|
regex_value =
|
|
"^.*:\\s(\\d\\d\\d\\d/\\d\\d/\\d\\d\\s\\d\\d:\\d\\d:\\d\\d)\\s.*$";
|
|
});
|
|
target_field = "timestamp";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "none";
|
|
converters = {
|
|
config = toJSON ({ date_format = "yyyy/MM/dd HH:mm:ss "; });
|
|
type = "date";
|
|
};
|
|
};
|
|
nginx_error_server = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
type = "regex";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "string";
|
|
condition_value = "server";
|
|
extractor_config = toJSON ({ regex_value = "server:\\s(.+?)(,|$)"; });
|
|
order = 1;
|
|
target_field = "server";
|
|
title = "server";
|
|
};
|
|
nginx_error_remote_addr = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
type = "regex";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "string";
|
|
condition_value = "client";
|
|
extractor_config = toJSON ({ regex_value = "client:\\s(.+?)(,|$)"; });
|
|
order = 2;
|
|
target_field = "remote_addr";
|
|
title = "remote_addr/client";
|
|
};
|
|
nginx_error_host = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
type = "regex";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "string";
|
|
condition_value = "host";
|
|
extractor_config = toJSON ({ regex_value = ''host:\s"(.+?)"(,|$)''; });
|
|
order = 3;
|
|
target_field = "host";
|
|
title = "host";
|
|
};
|
|
nginx_error_request_path = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
type = "regex";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "string";
|
|
condition_value = "request";
|
|
extractor_config =
|
|
toJSON ({ regex_value = ''request:\s"(.+?)"(,|$)''; });
|
|
order = 4;
|
|
target_field = "request_path";
|
|
title = "request_path/request";
|
|
};
|
|
nginx_error_request_verb = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
type = "regex";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "string";
|
|
condition_value = "request";
|
|
extractor_config = toJSON ({
|
|
regex_value = ''
|
|
request:\s"(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT|PATCH).+"(,|$)'';
|
|
});
|
|
order = 5;
|
|
target_field = "request_verb";
|
|
title = "request_verb";
|
|
};
|
|
|
|
# nginx access
|
|
nginx_access_json_from_syslog = {
|
|
input_id = "\${graylog_input.nginx_access_logs.id}";
|
|
title = "Get JSON from syslog message";
|
|
type = "regex";
|
|
cursor_strategy = "copy";
|
|
condition_type = "none";
|
|
source_field = "message";
|
|
target_field = "json";
|
|
order = 0;
|
|
extractor_config = toJSON ({ regex_value = "nginx:\\s+(.*)"; });
|
|
};
|
|
nginx_access_extract_json = {
|
|
input_id = "\${graylog_input.nginx_access_logs.id}";
|
|
title = "Extract JSON fields";
|
|
order = 1;
|
|
source_field = "json";
|
|
type = "json";
|
|
cursor_strategy = "copy";
|
|
condition_type = "none";
|
|
extractor_config = toJSON ({
|
|
flatten = true;
|
|
list_separator = ", ";
|
|
kv_separator = "=";
|
|
key_prefix = "";
|
|
key_separator = "_";
|
|
replace_key_whitespace = false;
|
|
key_whitespace_replacement = "_";
|
|
});
|
|
};
|
|
nginx_access_empty_json = {
|
|
input_id = "\${graylog_input.nginx_access_logs.id}";
|
|
order = 2;
|
|
title = "Empty JSON field";
|
|
type = "regex_replace";
|
|
extractor_config = toJSON ({
|
|
regex = ".*";
|
|
replacement = "-";
|
|
});
|
|
target_field = "json";
|
|
source_field = "json";
|
|
cursor_strategy = "copy";
|
|
condition_type = "none";
|
|
};
|
|
nginx_access_reduce_message = {
|
|
input_id = "\${graylog_input.nginx_access_logs.id}";
|
|
order = 3;
|
|
title = "Reduced message to path";
|
|
type = "regex_replace";
|
|
extractor_config = toJSON ({
|
|
regex = ''.*request": "(.*?)".*'';
|
|
replacement = "$1";
|
|
});
|
|
target_field = "message";
|
|
source_field = "message";
|
|
cursor_strategy = "copy";
|
|
condition_type = "none";
|
|
};
|
|
|
|
};
|
|
|
|
graylog_input_static_fields = {
|
|
|
|
nginx_access_logs = {
|
|
input_id = "\${graylog_input.nginx_access_logs.id}";
|
|
fields = {
|
|
from_nginx = true;
|
|
nginx_error = false;
|
|
nginx_access = true;
|
|
};
|
|
};
|
|
|
|
nginx_error_logs = {
|
|
input_id = "\${graylog_input.nginx_error_logs.id}";
|
|
fields = {
|
|
from_nginx = true;
|
|
nginx_error = true;
|
|
nginx_access = false;
|
|
};
|
|
};
|
|
|
|
};
|
|
|
|
graylog_stream = {
|
|
nginx5xx = {
|
|
title = "nginx 5xx";
|
|
description = "all requests answered with a 5xx response";
|
|
index_set_id = "\${graylog_index_set.default.id}";
|
|
disabled = false;
|
|
matching_type = "AND";
|
|
};
|
|
nginx4xx = {
|
|
title = "nginx 4xx";
|
|
description = "all requests answered with a 4xx response";
|
|
index_set_id = "\${graylog_index_set.default.id}";
|
|
disabled = false;
|
|
matching_type = "AND";
|
|
};
|
|
nginx2xx = {
|
|
title = "nginx 2xx";
|
|
description = "all requests answered with a 2xx response";
|
|
index_set_id = "\${graylog_index_set.default.id}";
|
|
disabled = false;
|
|
matching_type = "AND";
|
|
};
|
|
nginx_access = {
|
|
title = "nginx access";
|
|
description = "all requests";
|
|
index_set_id = "\${graylog_index_set.default.id}";
|
|
disabled = false;
|
|
matching_type = "AND";
|
|
};
|
|
nginx_error = {
|
|
title = "nginx error";
|
|
description = "all errors";
|
|
index_set_id = "\${graylog_index_set.default.id}";
|
|
disabled = false;
|
|
matching_type = "AND";
|
|
};
|
|
};
|
|
|
|
graylog_stream_rule =
|
|
let
|
|
nq_stream_rule = field: value: stream_id: {
|
|
inherit field value stream_id;
|
|
type = 1;
|
|
inverted = true;
|
|
};
|
|
eq_stream_rule = field: value: stream_id: {
|
|
inherit field value stream_id;
|
|
type = 1;
|
|
inverted = false;
|
|
};
|
|
gt_stream_rule = field: value: stream_id: {
|
|
inherit field value stream_id;
|
|
type = 3;
|
|
inverted = false;
|
|
};
|
|
lt_stream_rule = field: value: stream_id: {
|
|
inherit field value stream_id;
|
|
type = 4;
|
|
inverted = false;
|
|
};
|
|
between = min: max: stream_id: {
|
|
"is_nginx_access_${min}_${max}" =
|
|
(eq_stream_rule "nginx_access" true stream_id);
|
|
"nginx_above${min}" = (gt_stream_rule "response_status" min stream_id);
|
|
"nginx_below${max}" = (lt_stream_rule "response_status" max stream_id);
|
|
};
|
|
in
|
|
(between "499" "600" "\${graylog_stream.nginx5xx.id}")
|
|
// (between "399" "500" "\${graylog_stream.nginx4xx.id}")
|
|
// (between "199" "300" "\${graylog_stream.nginx2xx.id}") // {
|
|
is_nginx_access = (eq_stream_rule "nginx_access" true
|
|
"\${graylog_stream.nginx_access.id}");
|
|
is_nginx_error =
|
|
(eq_stream_rule "nginx_error" true "\${graylog_stream.nginx_error.id}");
|
|
};
|
|
|
|
};
|
|
}
|