palo/nixos-config
palo/nixos-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

graylog: fine tuning and dashboard creation

Browse source
This commit is contained in:
Ingolf Wagner 2021-07-17 13:45:19 +02:00
parent ebc9d34b78
commit f18e242afd
No known key found for this signature in database
GPG key ID: 76BF5F1928B9618B
9 changed files with 783 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
configs/workhorse/graylog.nix
View file

@ -24,9 +24,11 @@ in {
};
services.mongodb.enable = true;
services.elasticsearch.enable = true;
services.elasticsearch.listenAddress =
"${config.networking.hostName}.private";
services.elasticsearch = {
enable = true;
listenAddress = "${config.networking.hostName}.private";
extraJavaOptions = ["-Des.http.cname_in_publish_address=true"];
};
services.graylog.enable = true;
services.graylog.elasticsearchHosts =

2
configs/workhorse/nextcloud.nix
View file

@ -210,7 +210,7 @@ in {
autoUpdateApps.enable = true;
#nginx.enable = true;
hostName = "nextcloud.ingolf-wagner.de";
#logLevel = 0;
logLevel = 2;
https = true;
config = {
adminpassFile =

674
terranix/graylog/MyDashboards.json Normal file
View file

@ -0,0 +1,674 @@
{
"v": "1",
"id": "da023d7e-086a-4387-a5b1-02bd267d9c3f",
"rev": 2,
"name": "Dashboards",
"summary": "My Dashboards",
"description": "All my Dashboards focusing mainly on journald logs",
"vendor": "Ingolf Wagner",
"url": "",
"parameters": [],
"entities": [
{
"v": "1",
"type": {
"name": "dashboard",
"version": "2"
},
"id": "04d927ad-a217-43bf-aa9e-820777399cc3",
"data": {
"summary": {
"@type": "string",
"@value": "Overview on Graylog"
},
"search": {
"queries": [
{
"id": "bfb6a815-7213-484c-91ba-ebaeff542a66",
"timerange": {
"type": "relative",
"range": 300
},
"query": {
"type": "elasticsearch",
"query_string": ""
},
"search_types": [
{
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)"
},
"name": "chart",
"timerange": {
"type": "relative",
"range": 86400
},
"streams": [],
"series": [
{
"type": "count",
"id": "count()",
"field": null
}
],
"filter": null,
"rollup": false,
"row_groups": [
{
"type": "time",
"field": "timestamp",
"interval": {
"type": "auto",
"scaling": 1
}
}
],
"type": "pivot",
"id": "8e1ed6ed-ff1f-4d86-8981-a987aaaa5eed",
"column_groups": [
{
"type": "values",
"field": "systemd_unit",
"limit": 15
}
],
"sort": []
},
{
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true"
},
"name": "chart",
"timerange": {
"type": "relative",
"range": 86400
},
"streams": [],
"series": [
{
"type": "count",
"id": "count()",
"field": null
}
],
"filter": null,
"rollup": false,
"row_groups": [
{
"type": "time",
"field": "timestamp",
"interval": {
"type": "auto",
"scaling": 1
}
}
],
"type": "pivot",
"id": "d7e2a713-28fd-46d1-8c7a-29bd2867bebd",
"column_groups": [
{
"type": "values",
"field": "source",
"limit": 15
}
],
"sort": []
}
]
}
],
"parameters": [],
"requires": {},
"owner": "admin",
"created_at": "2021-07-17T08:03:26.960Z"
},
"created_at": "2021-07-17T05:53:41.503Z",
"requires": {},
"state": {
"bfb6a815-7213-484c-91ba-ebaeff542a66": {
"selected_fields": null,
"static_message_list_id": null,
"titles": {
"widget": {
"b0d1972c-c917-4054-a946-d412859ee5f0": "Graylog Errors of last day",
"49928524-8949-42e2-b6a6-4f208e2febb5": "Graylog Input of last day",
"c535afa8-b27f-4cec-b117-483df2d439ec": "Graylog errors of last day",
"9a6682e0-8993-439a-bfff-62e4a3c99473": "Graylog errors of last day (copy)"
},
"tab": {
"title": "Last Day"
}
},
"widgets": [
{
"id": "c535afa8-b27f-4cec-b117-483df2d439ec",
"type": "aggregation",
"filter": null,
"timerange": {
"type": "relative",
"range": 86400
},
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)"
},
"streams": [],
"config": {
"visualization": "line",
"event_annotation": false,
"row_pivots": [
{
"field": "timestamp",
"type": "time",
"config": {
"interval": {
"type": "auto",
"scaling": null
}
}
}
],
"series": [
{
"config": {
"name": null
},
"function": "count()"
}
],
"rollup": false,
"column_pivots": [
{
"field": "systemd_unit",
"type": "values",
"config": {
"limit": 15
}
}
],
"visualization_config": {
"interpolation": "spline"
},
"formatting_settings": null,
"sort": []
}
},
{
"id": "49928524-8949-42e2-b6a6-4f208e2febb5",
"type": "aggregation",
"filter": null,
"timerange": {
"type": "relative",
"range": 86400
},
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true"
},
"streams": [],
"config": {
"visualization": "line",
"event_annotation": false,
"row_pivots": [
{
"field": "timestamp",
"type": "time",
"config": {
"interval": {
"type": "auto",
"scaling": null
}
}
}
],
"series": [
{
"config": {
"name": null
},
"function": "count()"
}
],
"rollup": false,
"column_pivots": [
{
"field": "source",
"type": "values",
"config": {
"limit": 15
}
}
],
"visualization_config": {
"interpolation": "spline"
},
"formatting_settings": null,
"sort": []
}
}
],
"widget_mapping": {
"49928524-8949-42e2-b6a6-4f208e2febb5": [
"d7e2a713-28fd-46d1-8c7a-29bd2867bebd"
],
"c535afa8-b27f-4cec-b117-483df2d439ec": [
"8e1ed6ed-ff1f-4d86-8981-a987aaaa5eed"
]
},
"positions": {
"49928524-8949-42e2-b6a6-4f208e2febb5": {
"col": 1,
"row": 11,
"height": 3,
"width": "Infinity"
},
"c535afa8-b27f-4cec-b117-483df2d439ec": {
"col": 1,
"row": 8,
"height": 3,
"width": "Infinity"
}
},
"formatting": {
"highlighting": []
},
"display_mode_settings": {
"positions": {}
}
}
},
"properties": [],
"owner": "admin",
"title": {
"@type": "string",
"@value": "Graylog"
},
"type": "DASHBOARD",
"description": {
"@type": "string",
"@value": ""
}
},
"constraints": [
{
"type": "server-version",
"version": ">=3.3.9+abab7dc"
}
]
},
{
"v": "1",
"type": {
"name": "dashboard",
"version": "2"
},
"id": "40d84ea8-3f72-47b8-9819-722b3f5dcbd3",
"data": {
"summary": {
"@type": "string",
"@value": "Overview on Graylog"
},
"search": {
"queries": [
{
"id": "bfb6a815-7213-484c-91ba-ebaeff542a66",
"timerange": {
"type": "relative",
"range": 300
},
"query": {
"type": "elasticsearch",
"query_string": ""
},
"search_types": [
{
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND systemd_unit:init.scope AND syslog_priority:4"
},
"name": "chart",
"timerange": {
"type": "relative",
"range": 86400
},
"streams": [],
"series": [
{
"type": "count",
"id": "count()",
"field": null
}
],
"filter": null,
"rollup": true,
"row_groups": [
{
"type": "values",
"field": "custom_unit",
"limit": 15
}
],
"type": "pivot",
"id": "d480b368-2968-442c-94b9-e1e4e1830db7",
"column_groups": [],
"sort": []
},
{
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true"
},
"name": "chart",
"timerange": {
"type": "relative",
"range": 86400
},
"streams": [],
"series": [
{
"type": "count",
"id": "count()",
"field": null
}
],
"filter": null,
"rollup": false,
"row_groups": [
{
"type": "time",
"field": "timestamp",
"interval": {
"type": "auto",
"scaling": 1
}
}
],
"type": "pivot",
"id": "148df0da-281a-4266-a363-9565c9b851b6",
"column_groups": [
{
"type": "values",
"field": "source",
"limit": 15
}
],
"sort": []
},
{
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)"
},
"name": "chart",
"timerange": {
"type": "relative",
"range": 86400
},
"streams": [],
"series": [
{
"type": "count",
"id": "count()",
"field": null
}
],
"filter": null,
"rollup": false,
"row_groups": [
{
"type": "time",
"field": "timestamp",
"interval": {
"type": "auto",
"scaling": 1
}
}
],
"type": "pivot",
"id": "fe958d96-6908-4516-848d-9490d810ed3e",
"column_groups": [
{
"type": "values",
"field": "systemd_unit",
"limit": 15
}
],
"sort": []
}
]
}
],
"parameters": [],
"requires": {},
"owner": "admin",
"created_at": "2021-07-17T11:41:39.203Z"
},
"created_at": "2021-07-17T05:53:41.503Z",
"requires": {},
"state": {
"bfb6a815-7213-484c-91ba-ebaeff542a66": {
"selected_fields": null,
"static_message_list_id": null,
"titles": {
"widget": {
"b0d1972c-c917-4054-a946-d412859ee5f0": "Graylog Errors of last day",
"49928524-8949-42e2-b6a6-4f208e2febb5": "Graylog Input of last day",
"c535afa8-b27f-4cec-b117-483df2d439ec": "Graylog errors of last day",
"9a6682e0-8993-439a-bfff-62e4a3c99473": "Graylog errors of last day (copy)",
"ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": "init.scope warnings",
"221557b8-5b8b-4c57-9449-00a1aaf91388": "Messages for custom_unit:backup.mount"
},
"tab": {
"title": "Last Day"
}
},
"widgets": [
{
"id": "c535afa8-b27f-4cec-b117-483df2d439ec",
"type": "aggregation",
"filter": null,
"timerange": {
"type": "relative",
"range": 86400
},
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)"
},
"streams": [],
"config": {
"visualization": "line",
"event_annotation": false,
"row_pivots": [
{
"field": "timestamp",
"type": "time",
"config": {
"interval": {
"type": "auto",
"scaling": null
}
}
}
],
"series": [
{
"config": {
"name": null
},
"function": "count()"
}
],
"rollup": false,
"column_pivots": [
{
"field": "systemd_unit",
"type": "values",
"config": {
"limit": 15
}
}
],
"visualization_config": {
"interpolation": "spline"
},
"formatting_settings": null,
"sort": []
}
},
{
"id": "49928524-8949-42e2-b6a6-4f208e2febb5",
"type": "aggregation",
"filter": null,
"timerange": {
"type": "relative",
"range": 86400
},
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true"
},
"streams": [],
"config": {
"visualization": "line",
"event_annotation": false,
"row_pivots": [
{
"field": "timestamp",
"type": "time",
"config": {
"interval": {
"type": "auto",
"scaling": null
}
}
}
],
"series": [
{
"config": {
"name": null
},
"function": "count()"
}
],
"rollup": false,
"column_pivots": [
{
"field": "source",
"type": "values",
"config": {
"limit": 15
}
}
],
"visualization_config": {
"interpolation": "spline"
},
"formatting_settings": null,
"sort": []
}
},
{
"id": "ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa",
"type": "aggregation",
"filter": null,
"timerange": {
"type": "relative",
"range": 86400
},
"query": {
"type": "elasticsearch",
"query_string": "from_journald:true AND systemd_unit:init.scope AND syslog_priority:4"
},
"streams": [],
"config": {
"visualization": "table",
"event_annotation": false,
"row_pivots": [
{
"field": "custom_unit",
"type": "values",
"config": {
"limit": 15
}
}
],
"series": [
{
"config": {
"name": null
},
"function": "count()"
}
],
"rollup": true,
"column_pivots": [],
"visualization_config": null,
"formatting_settings": null,
"sort": []
}
}
],
"widget_mapping": {
"c535afa8-b27f-4cec-b117-483df2d439ec": [
"fe958d96-6908-4516-848d-9490d810ed3e"
],
"ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": [
"d480b368-2968-442c-94b9-e1e4e1830db7"
],
"49928524-8949-42e2-b6a6-4f208e2febb5": [
"148df0da-281a-4266-a363-9565c9b851b6"
]
},
"positions": {
"ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": {
"col": 1,
"row": 6,
"height": 6,
"width": 4
},
"c535afa8-b27f-4cec-b117-483df2d439ec": {
"col": 5,
"row": 6,
"height": 3,
"width": 8
},
"49928524-8949-42e2-b6a6-4f208e2febb5": {
"col": 5,
"row": 9,
"height": 3,
"width": 8
}
},
"formatting": {
"highlighting": []
},
"display_mode_settings": {
"positions": {}
}
}
},
"properties": [],
"owner": "admin",
"title": {
"@type": "string",
"@value": "Graylog"
},
"type": "DASHBOARD",
"description": {
"@type": "string",
"@value": ""
}
},
"constraints": [
{
"type": "server-version",
"version": ">=3.3.9+abab7dc"
}
]
}
]
}

4
terranix/graylog/config.nix
View file

@ -24,19 +24,17 @@ with builtins; {
retention_strategy_class =
"org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy";
index_analyzer = "standard";
index_optimization_disabled = true;
index_optimization_disabled = false;
writable = true;
shards = 1;
replicas = 0;
index_optimization_max_num_segments = 1;
field_type_refresh_interval = 5000;
retention_strategy = toJSON ({
max_number_of_indices = maxIndexCount;
type =
"org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig";
});
rotation_strategy = toJSON ({
#max_docs_per_index = 30000000;
max_size = maxIndexSize * 1024 * 1024;

2
terranix/graylog/journald.nix
View file

@ -1,6 +1,6 @@
with builtins; {
imports = [ ./journald/nextcloud.nix ];
imports = [ ./journald/nextcloud.nix ./journald/kibana.nix ];
resource = {

59
terranix/graylog/journald/kibana.nix Normal file
View file

@ -0,0 +1,59 @@
with builtins; {
resource = {
graylog_pipeline_connection = {
journald.pipeline_ids = [ "\${graylog_pipeline.kibana.id}" ];
};
graylog_pipeline = {
kibana.source = ''
pipeline "kibana : parsing"
stage 10 match either
rule "kibana : parse level 1"
stage 11 match either
rule "kibana : parse message"
end
'';
};
graylog_pipeline_rule = {
kibanaLevel1.source = ''
rule "kibana : parse level 1"
when
has_field("systemd_unit") && ($message.systemd_unit == "kibana.service")
then
let parsedJson = parse_json(to_string($message.message));
set_fields(to_map(parsedJson),"kibana_");
end
'';
kibanaLevelRequest.source = ''
rule "kibana : parse request"
when
has_field("kibana_req")
then
let parsedJson = parse_json(to_string($message.kibana_req));
set_fields(to_map(parsedJson),"kibana_req_");
end
'';
kibanaLevelResponse.source = ''
rule "kibana : parse response"
when
has_field("kibana_res")
then
let parsedJson = parse_json(to_string($message.kibana_res));
set_fields(to_map(parsedJson),"kibana_res_");
end
'';
kibanaLevelMessage.source = ''
rule "kibana : parse message"
when
has_field("kibana_message")
then
set_field("message", $message.kibana_message);
end
'';
};
};
}

26
terranix/graylog/journald/nextcloud.nix
View file

@ -13,6 +13,8 @@ with builtins; {
rule "nextcloud : parse level 1"
stage 11 match either
rule "nextcloud : parse level 2"
stage 12 match either
rule "nextcloud : parse level 3"
end
'';
};
@ -21,19 +23,37 @@ with builtins; {
nextcloudLevel1.source = ''
rule "nextcloud : parse level 1"
when
has_field("systemd_unit") && $message.systemd_unit == "phpfpm-nextcloud.service"
has_field("systemd_unit") && ($message.systemd_unit == "phpfpm-nextcloud.service" || $message.systemd_unit == "nextcloud-cron.service") && starts_with(to_string($message.message),"{")
then
let parsedJson = parse_json(to_string($message.message));
set_fields(to_map(parsedJson),"nextcloud_");
end
'';
#nextcloudLevel2.source = ''
# rule "nextcloud : parse level 2"
# when
# has_field("nextcloud_message")
# then
# let parsedJson = parse_json(to_string($message.nextcloud_message));
# set_field("message", $message.nextcloud_message);
# set_fields(to_map(parsedJson),"nextcloud_message_");
# end
#'';
nextcloudLevel2.source = ''
rule "nextcloud : parse level 2"
when
has_field("nextcloud_message")
then
let parsedJson = parse_json(to_string($message.nextcloud_message));
set_fields(to_map(parsedJson),"nextcloud_message_");
set_field("message", $message.nextcloud_message);
end
'';
nextcloudLevel3.source = ''
rule "nextcloud : parse level 3"
when
has_field("nextcloud_message_Message")
then
remove_field("nextcloud_message");
set_field("message", $message.nextcloud_message_Message);
end
'';
};

6
terranix/graylog/nginx.nix
View file

@ -1,6 +1,4 @@
/*
# use this nginx configuration
/* # use this nginx configuration
# to send data to these inputs
log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
@ -20,12 +18,10 @@ log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
access_log syslog:server=${access_log_input} graylog2_json;
error_log syslog:server=${error_log_input};
*/
with builtins; {
resource = {
graylog_input = {

BIN
terranix/graylog/terraform.tfstate
View file

Binary file not shown.