diff --git a/configs/workhorse/graylog.nix b/configs/workhorse/graylog.nix index f2c87c0..507edd9 100644 --- a/configs/workhorse/graylog.nix +++ b/configs/workhorse/graylog.nix @@ -24,9 +24,11 @@ in { }; services.mongodb.enable = true; - services.elasticsearch.enable = true; - services.elasticsearch.listenAddress = - "${config.networking.hostName}.private"; + services.elasticsearch = { + enable = true; + listenAddress = "${config.networking.hostName}.private"; + extraJavaOptions = ["-Des.http.cname_in_publish_address=true"]; + }; services.graylog.enable = true; services.graylog.elasticsearchHosts = diff --git a/configs/workhorse/nextcloud.nix b/configs/workhorse/nextcloud.nix index ee0de2e..a6033c5 100644 --- a/configs/workhorse/nextcloud.nix +++ b/configs/workhorse/nextcloud.nix @@ -210,7 +210,7 @@ in { autoUpdateApps.enable = true; #nginx.enable = true; hostName = "nextcloud.ingolf-wagner.de"; - #logLevel = 0; + logLevel = 2; https = true; config = { adminpassFile = diff --git a/terranix/graylog/MyDashboards.json b/terranix/graylog/MyDashboards.json new file mode 100644 index 0000000..5df9da1 --- /dev/null +++ b/terranix/graylog/MyDashboards.json @@ -0,0 +1,674 @@ +{ + "v": "1", + "id": "da023d7e-086a-4387-a5b1-02bd267d9c3f", + "rev": 2, + "name": "Dashboards", + "summary": "My Dashboards", + "description": "All my Dashboards focusing mainly on journald logs", + "vendor": "Ingolf Wagner", + "url": "", + "parameters": [], + "entities": [ + { + "v": "1", + "type": { + "name": "dashboard", + "version": "2" + }, + "id": "04d927ad-a217-43bf-aa9e-820777399cc3", + "data": { + "summary": { + "@type": "string", + "@value": "Overview on Graylog" + }, + "search": { + "queries": [ + { + "id": "bfb6a815-7213-484c-91ba-ebaeff542a66", + "timerange": { + "type": "relative", + "range": 300 + }, + "query": { + "type": "elasticsearch", + "query_string": "" + }, + "search_types": [ + { + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)" + }, + "name": "chart", + "timerange": { + "type": "relative", + "range": 86400 + }, + "streams": [], + "series": [ + { + "type": "count", + "id": "count()", + "field": null + } + ], + "filter": null, + "rollup": false, + "row_groups": [ + { + "type": "time", + "field": "timestamp", + "interval": { + "type": "auto", + "scaling": 1 + } + } + ], + "type": "pivot", + "id": "8e1ed6ed-ff1f-4d86-8981-a987aaaa5eed", + "column_groups": [ + { + "type": "values", + "field": "systemd_unit", + "limit": 15 + } + ], + "sort": [] + }, + { + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true" + }, + "name": "chart", + "timerange": { + "type": "relative", + "range": 86400 + }, + "streams": [], + "series": [ + { + "type": "count", + "id": "count()", + "field": null + } + ], + "filter": null, + "rollup": false, + "row_groups": [ + { + "type": "time", + "field": "timestamp", + "interval": { + "type": "auto", + "scaling": 1 + } + } + ], + "type": "pivot", + "id": "d7e2a713-28fd-46d1-8c7a-29bd2867bebd", + "column_groups": [ + { + "type": "values", + "field": "source", + "limit": 15 + } + ], + "sort": [] + } + ] + } + ], + "parameters": [], + "requires": {}, + "owner": "admin", + "created_at": "2021-07-17T08:03:26.960Z" + }, + "created_at": "2021-07-17T05:53:41.503Z", + "requires": {}, + "state": { + "bfb6a815-7213-484c-91ba-ebaeff542a66": { + "selected_fields": null, + "static_message_list_id": null, + "titles": { + "widget": { + "b0d1972c-c917-4054-a946-d412859ee5f0": "Graylog Errors of last day", + "49928524-8949-42e2-b6a6-4f208e2febb5": "Graylog Input of last day", + "c535afa8-b27f-4cec-b117-483df2d439ec": "Graylog errors of last day", + "9a6682e0-8993-439a-bfff-62e4a3c99473": "Graylog errors of last day (copy)" + }, + "tab": { + "title": "Last Day" + } + }, + "widgets": [ + { + "id": "c535afa8-b27f-4cec-b117-483df2d439ec", + "type": "aggregation", + "filter": null, + "timerange": { + "type": "relative", + "range": 86400 + }, + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)" + }, + "streams": [], + "config": { + "visualization": "line", + "event_annotation": false, + "row_pivots": [ + { + "field": "timestamp", + "type": "time", + "config": { + "interval": { + "type": "auto", + "scaling": null + } + } + } + ], + "series": [ + { + "config": { + "name": null + }, + "function": "count()" + } + ], + "rollup": false, + "column_pivots": [ + { + "field": "systemd_unit", + "type": "values", + "config": { + "limit": 15 + } + } + ], + "visualization_config": { + "interpolation": "spline" + }, + "formatting_settings": null, + "sort": [] + } + }, + { + "id": "49928524-8949-42e2-b6a6-4f208e2febb5", + "type": "aggregation", + "filter": null, + "timerange": { + "type": "relative", + "range": 86400 + }, + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true" + }, + "streams": [], + "config": { + "visualization": "line", + "event_annotation": false, + "row_pivots": [ + { + "field": "timestamp", + "type": "time", + "config": { + "interval": { + "type": "auto", + "scaling": null + } + } + } + ], + "series": [ + { + "config": { + "name": null + }, + "function": "count()" + } + ], + "rollup": false, + "column_pivots": [ + { + "field": "source", + "type": "values", + "config": { + "limit": 15 + } + } + ], + "visualization_config": { + "interpolation": "spline" + }, + "formatting_settings": null, + "sort": [] + } + } + ], + "widget_mapping": { + "49928524-8949-42e2-b6a6-4f208e2febb5": [ + "d7e2a713-28fd-46d1-8c7a-29bd2867bebd" + ], + "c535afa8-b27f-4cec-b117-483df2d439ec": [ + "8e1ed6ed-ff1f-4d86-8981-a987aaaa5eed" + ] + }, + "positions": { + "49928524-8949-42e2-b6a6-4f208e2febb5": { + "col": 1, + "row": 11, + "height": 3, + "width": "Infinity" + }, + "c535afa8-b27f-4cec-b117-483df2d439ec": { + "col": 1, + "row": 8, + "height": 3, + "width": "Infinity" + } + }, + "formatting": { + "highlighting": [] + }, + "display_mode_settings": { + "positions": {} + } + } + }, + "properties": [], + "owner": "admin", + "title": { + "@type": "string", + "@value": "Graylog" + }, + "type": "DASHBOARD", + "description": { + "@type": "string", + "@value": "" + } + }, + "constraints": [ + { + "type": "server-version", + "version": ">=3.3.9+abab7dc" + } + ] + }, + { + "v": "1", + "type": { + "name": "dashboard", + "version": "2" + }, + "id": "40d84ea8-3f72-47b8-9819-722b3f5dcbd3", + "data": { + "summary": { + "@type": "string", + "@value": "Overview on Graylog" + }, + "search": { + "queries": [ + { + "id": "bfb6a815-7213-484c-91ba-ebaeff542a66", + "timerange": { + "type": "relative", + "range": 300 + }, + "query": { + "type": "elasticsearch", + "query_string": "" + }, + "search_types": [ + { + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND systemd_unit:init.scope AND syslog_priority:4" + }, + "name": "chart", + "timerange": { + "type": "relative", + "range": 86400 + }, + "streams": [], + "series": [ + { + "type": "count", + "id": "count()", + "field": null + } + ], + "filter": null, + "rollup": true, + "row_groups": [ + { + "type": "values", + "field": "custom_unit", + "limit": 15 + } + ], + "type": "pivot", + "id": "d480b368-2968-442c-94b9-e1e4e1830db7", + "column_groups": [], + "sort": [] + }, + { + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true" + }, + "name": "chart", + "timerange": { + "type": "relative", + "range": 86400 + }, + "streams": [], + "series": [ + { + "type": "count", + "id": "count()", + "field": null + } + ], + "filter": null, + "rollup": false, + "row_groups": [ + { + "type": "time", + "field": "timestamp", + "interval": { + "type": "auto", + "scaling": 1 + } + } + ], + "type": "pivot", + "id": "148df0da-281a-4266-a363-9565c9b851b6", + "column_groups": [ + { + "type": "values", + "field": "source", + "limit": 15 + } + ], + "sort": [] + }, + { + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)" + }, + "name": "chart", + "timerange": { + "type": "relative", + "range": 86400 + }, + "streams": [], + "series": [ + { + "type": "count", + "id": "count()", + "field": null + } + ], + "filter": null, + "rollup": false, + "row_groups": [ + { + "type": "time", + "field": "timestamp", + "interval": { + "type": "auto", + "scaling": 1 + } + } + ], + "type": "pivot", + "id": "fe958d96-6908-4516-848d-9490d810ed3e", + "column_groups": [ + { + "type": "values", + "field": "systemd_unit", + "limit": 15 + } + ], + "sort": [] + } + ] + } + ], + "parameters": [], + "requires": {}, + "owner": "admin", + "created_at": "2021-07-17T11:41:39.203Z" + }, + "created_at": "2021-07-17T05:53:41.503Z", + "requires": {}, + "state": { + "bfb6a815-7213-484c-91ba-ebaeff542a66": { + "selected_fields": null, + "static_message_list_id": null, + "titles": { + "widget": { + "b0d1972c-c917-4054-a946-d412859ee5f0": "Graylog Errors of last day", + "49928524-8949-42e2-b6a6-4f208e2febb5": "Graylog Input of last day", + "c535afa8-b27f-4cec-b117-483df2d439ec": "Graylog errors of last day", + "9a6682e0-8993-439a-bfff-62e4a3c99473": "Graylog errors of last day (copy)", + "ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": "init.scope warnings", + "221557b8-5b8b-4c57-9449-00a1aaf91388": "Messages for custom_unit:backup.mount" + }, + "tab": { + "title": "Last Day" + } + }, + "widgets": [ + { + "id": "c535afa8-b27f-4cec-b117-483df2d439ec", + "type": "aggregation", + "filter": null, + "timerange": { + "type": "relative", + "range": 86400 + }, + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND syslog_facility:<4 AND (systemd_unit:elasticsearch.service OR systemd_unit:kibana.service OR systemd_unit:graylog.service)" + }, + "streams": [], + "config": { + "visualization": "line", + "event_annotation": false, + "row_pivots": [ + { + "field": "timestamp", + "type": "time", + "config": { + "interval": { + "type": "auto", + "scaling": null + } + } + } + ], + "series": [ + { + "config": { + "name": null + }, + "function": "count()" + } + ], + "rollup": false, + "column_pivots": [ + { + "field": "systemd_unit", + "type": "values", + "config": { + "limit": 15 + } + } + ], + "visualization_config": { + "interpolation": "spline" + }, + "formatting_settings": null, + "sort": [] + } + }, + { + "id": "49928524-8949-42e2-b6a6-4f208e2febb5", + "type": "aggregation", + "filter": null, + "timerange": { + "type": "relative", + "range": 86400 + }, + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true" + }, + "streams": [], + "config": { + "visualization": "line", + "event_annotation": false, + "row_pivots": [ + { + "field": "timestamp", + "type": "time", + "config": { + "interval": { + "type": "auto", + "scaling": null + } + } + } + ], + "series": [ + { + "config": { + "name": null + }, + "function": "count()" + } + ], + "rollup": false, + "column_pivots": [ + { + "field": "source", + "type": "values", + "config": { + "limit": 15 + } + } + ], + "visualization_config": { + "interpolation": "spline" + }, + "formatting_settings": null, + "sort": [] + } + }, + { + "id": "ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa", + "type": "aggregation", + "filter": null, + "timerange": { + "type": "relative", + "range": 86400 + }, + "query": { + "type": "elasticsearch", + "query_string": "from_journald:true AND systemd_unit:init.scope AND syslog_priority:4" + }, + "streams": [], + "config": { + "visualization": "table", + "event_annotation": false, + "row_pivots": [ + { + "field": "custom_unit", + "type": "values", + "config": { + "limit": 15 + } + } + ], + "series": [ + { + "config": { + "name": null + }, + "function": "count()" + } + ], + "rollup": true, + "column_pivots": [], + "visualization_config": null, + "formatting_settings": null, + "sort": [] + } + } + ], + "widget_mapping": { + "c535afa8-b27f-4cec-b117-483df2d439ec": [ + "fe958d96-6908-4516-848d-9490d810ed3e" + ], + "ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": [ + "d480b368-2968-442c-94b9-e1e4e1830db7" + ], + "49928524-8949-42e2-b6a6-4f208e2febb5": [ + "148df0da-281a-4266-a363-9565c9b851b6" + ] + }, + "positions": { + "ac9ffdfc-8f48-4ed8-af3b-62120dc86bfa": { + "col": 1, + "row": 6, + "height": 6, + "width": 4 + }, + "c535afa8-b27f-4cec-b117-483df2d439ec": { + "col": 5, + "row": 6, + "height": 3, + "width": 8 + }, + "49928524-8949-42e2-b6a6-4f208e2febb5": { + "col": 5, + "row": 9, + "height": 3, + "width": 8 + } + }, + "formatting": { + "highlighting": [] + }, + "display_mode_settings": { + "positions": {} + } + } + }, + "properties": [], + "owner": "admin", + "title": { + "@type": "string", + "@value": "Graylog" + }, + "type": "DASHBOARD", + "description": { + "@type": "string", + "@value": "" + } + }, + "constraints": [ + { + "type": "server-version", + "version": ">=3.3.9+abab7dc" + } + ] + } + ] +} \ No newline at end of file diff --git a/terranix/graylog/config.nix b/terranix/graylog/config.nix index 46e8c58..2bf8371 100644 --- a/terranix/graylog/config.nix +++ b/terranix/graylog/config.nix @@ -24,19 +24,17 @@ with builtins; { retention_strategy_class = "org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy"; index_analyzer = "standard"; - index_optimization_disabled = true; + index_optimization_disabled = false; writable = true; shards = 1; replicas = 0; index_optimization_max_num_segments = 1; field_type_refresh_interval = 5000; - retention_strategy = toJSON ({ max_number_of_indices = maxIndexCount; type = "org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig"; }); - rotation_strategy = toJSON ({ #max_docs_per_index = 30000000; max_size = maxIndexSize * 1024 * 1024; diff --git a/terranix/graylog/journald.nix b/terranix/graylog/journald.nix index 2d4edd6..5d6c872 100644 --- a/terranix/graylog/journald.nix +++ b/terranix/graylog/journald.nix @@ -1,6 +1,6 @@ with builtins; { - imports = [ ./journald/nextcloud.nix ]; + imports = [ ./journald/nextcloud.nix ./journald/kibana.nix ]; resource = { diff --git a/terranix/graylog/journald/kibana.nix b/terranix/graylog/journald/kibana.nix new file mode 100644 index 0000000..e6e856c --- /dev/null +++ b/terranix/graylog/journald/kibana.nix @@ -0,0 +1,59 @@ +with builtins; { + + resource = { + + graylog_pipeline_connection = { + journald.pipeline_ids = [ "\${graylog_pipeline.kibana.id}" ]; + }; + + graylog_pipeline = { + kibana.source = '' + pipeline "kibana : parsing" + stage 10 match either + rule "kibana : parse level 1" + stage 11 match either + rule "kibana : parse message" + end + ''; + }; + + graylog_pipeline_rule = { + kibanaLevel1.source = '' + rule "kibana : parse level 1" + when + has_field("systemd_unit") && ($message.systemd_unit == "kibana.service") + then + let parsedJson = parse_json(to_string($message.message)); + set_fields(to_map(parsedJson),"kibana_"); + end + ''; + kibanaLevelRequest.source = '' + rule "kibana : parse request" + when + has_field("kibana_req") + then + let parsedJson = parse_json(to_string($message.kibana_req)); + set_fields(to_map(parsedJson),"kibana_req_"); + end + ''; + kibanaLevelResponse.source = '' + rule "kibana : parse response" + when + has_field("kibana_res") + then + let parsedJson = parse_json(to_string($message.kibana_res)); + set_fields(to_map(parsedJson),"kibana_res_"); + end + ''; + kibanaLevelMessage.source = '' + rule "kibana : parse message" + when + has_field("kibana_message") + then + set_field("message", $message.kibana_message); + end + ''; + }; + + }; +} diff --git a/terranix/graylog/journald/nextcloud.nix b/terranix/graylog/journald/nextcloud.nix index 3a76e1f..e3e56ce 100644 --- a/terranix/graylog/journald/nextcloud.nix +++ b/terranix/graylog/journald/nextcloud.nix @@ -13,6 +13,8 @@ with builtins; { rule "nextcloud : parse level 1" stage 11 match either rule "nextcloud : parse level 2" + stage 12 match either + rule "nextcloud : parse level 3" end ''; }; @@ -21,19 +23,37 @@ with builtins; { nextcloudLevel1.source = '' rule "nextcloud : parse level 1" when - has_field("systemd_unit") && $message.systemd_unit == "phpfpm-nextcloud.service" + has_field("systemd_unit") && ($message.systemd_unit == "phpfpm-nextcloud.service" || $message.systemd_unit == "nextcloud-cron.service") && starts_with(to_string($message.message),"{") then let parsedJson = parse_json(to_string($message.message)); set_fields(to_map(parsedJson),"nextcloud_"); end ''; + #nextcloudLevel2.source = '' + # rule "nextcloud : parse level 2" + # when + # has_field("nextcloud_message") + # then + # let parsedJson = parse_json(to_string($message.nextcloud_message)); + # set_field("message", $message.nextcloud_message); + # set_fields(to_map(parsedJson),"nextcloud_message_"); + # end + #''; nextcloudLevel2.source = '' rule "nextcloud : parse level 2" when - has_field("nextcloud_message") + has_field("nextcloud_message") then - let parsedJson = parse_json(to_string($message.nextcloud_message)); - set_fields(to_map(parsedJson),"nextcloud_message_"); + set_field("message", $message.nextcloud_message); + end + ''; + nextcloudLevel3.source = '' + rule "nextcloud : parse level 3" + when + has_field("nextcloud_message_Message") + then + remove_field("nextcloud_message"); + set_field("message", $message.nextcloud_message_Message); end ''; }; diff --git a/terranix/graylog/nginx.nix b/terranix/graylog/nginx.nix index 772a107..b2b77f1 100644 --- a/terranix/graylog/nginx.nix +++ b/terranix/graylog/nginx.nix @@ -1,31 +1,27 @@ -/* +/* # use this nginx configuration + # to send data to these inputs -# use this nginx configuration -# to send data to these inputs - -log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", ' - '"facility": "nginx", ' - '"src_addr": "$remote_addr", ' - '"body_bytes_sent": $body_bytes_sent, ' - '"request_time": $request_time, ' - '"response_status": $status, ' - '"request": "$request", ' - '"request_method": "$request_method", ' - '"host": "$host",' - '"upstream_cache_status": "$upstream_cache_status",' - '"upstream_addr": "$upstream_addr",' - '"http_x_forwarded_for": "$http_x_forwarded_for",' - '"http_referrer": "$http_referer", ' - '"http_user_agent": "$http_user_agent" }'; - -access_log syslog:server=${access_log_input} graylog2_json; -error_log syslog:server=${error_log_input}; + log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", ' + '"facility": "nginx", ' + '"src_addr": "$remote_addr", ' + '"body_bytes_sent": $body_bytes_sent, ' + '"request_time": $request_time, ' + '"response_status": $status, ' + '"request": "$request", ' + '"request_method": "$request_method", ' + '"host": "$host",' + '"upstream_cache_status": "$upstream_cache_status",' + '"upstream_addr": "$upstream_addr",' + '"http_x_forwarded_for": "$http_x_forwarded_for",' + '"http_referrer": "$http_referer", ' + '"http_user_agent": "$http_user_agent" }'; + access_log syslog:server=${access_log_input} graylog2_json; + error_log syslog:server=${error_log_input}; */ with builtins; { - resource = { graylog_input = { diff --git a/terranix/graylog/terraform.tfstate b/terranix/graylog/terraform.tfstate index bc9b99b..4b0218c 100644 Binary files a/terranix/graylog/terraform.tfstate and b/terranix/graylog/terraform.tfstate differ