graylog: working on nginx

terranix/graylog/nginx.nix

@@ -40,6 +40,153 @@ with builtins; {
 
     };
 
+    graylog_extractor = {
+
+      # nginx error
+      nginx_error_timestamp = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        order = 0;
+        title = "Timestamp";
+        type = "regex";
+        extractor_config = toJSON ({
+          regex_value =
+            "^.*:\\s(\\d\\d\\d\\d/\\d\\d/\\d\\d\\s\\d\\d:\\d\\d:\\d\\d)\\s.*$";
+        });
+        target_field = "timestamp";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "none";
+        converters = {
+          config = toJSON ({ date_format = "yyyy/MM/dd HH:mm:ss "; });
+          type = "date";
+        };
+      };
+      nginx_error_server = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        type = "regex";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "string";
+        condition_value = "server";
+        extractor_config = toJSON ({ regex_value = "server:\\s(.+?)(,|$)"; });
+        order = 1;
+        target_field = "server";
+        title = "server";
+      };
+      nginx_error_remote_addr = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        type = "regex";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "string";
+        condition_value = "client";
+        extractor_config = toJSON ({ regex_value = "client:\\s(.+?)(,|$)"; });
+        order = 2;
+        target_field = "remote_addr";
+        title = "remote_addr/client";
+      };
+      nginx_error_host = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        type = "regex";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "string";
+        condition_value = "host";
+        extractor_config = toJSON ({ regex_value = ''host:\s"(.+?)"(,|$)''; });
+        order = 3;
+        target_field = "host";
+        title = "host";
+      };
+      nginx_error_request_path = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        type = "regex";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "string";
+        condition_value = "request";
+        extractor_config =
+          toJSON ({ regex_value = ''request:\s"(.+?)"(,|$)''; });
+        order = 4;
+        target_field = "request_path";
+        title = "request_path/request";
+      };
+      nginx_error_request_verb = {
+        input_id = "\${graylog_input.nginx_error_logs.id}";
+        type = "regex";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "string";
+        condition_value = "request";
+        extractor_config = toJSON ({
+          regex_value = ''
+            request:\s"(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT|PATCH).+"(,|$)'';
+        });
+        order = 5;
+        target_field = "request_verb";
+        title = "request_verb";
+      };
+
+      # nginx access
+      nginx_access_json_from_syslog = {
+        input_id = "\${graylog_input.nginx_access_logs.id}";
+        title = "Get JSON from syslog message";
+        type = "regex";
+        cursor_strategy = "copy";
+        condition_type = "none";
+        source_field = "message";
+        target_field = "json";
+        order = 0;
+        extractor_config = toJSON ({ regex_value = "nginx:\\s+(.*)"; });
+      };
+      nginx_access_extract_json = {
+        input_id = "\${graylog_input.nginx_access_logs.id}";
+        title = "Extract JSON fields";
+        order = 1;
+        source_field = "json";
+        type = "json";
+        cursor_strategy = "copy";
+        condition_type = "none";
+        extractor_config = toJSON ({
+          flatten = true;
+          list_separator = ", ";
+          kv_separator = "=";
+          key_prefix = "";
+          key_separator = "_";
+          replace_key_whitespace = false;
+          key_whitespace_replacement = "_";
+        });
+      };
+      nginx_access_empty_json = {
+        input_id = "\${graylog_input.nginx_access_logs.id}";
+        order = 2;
+        title = "Empty JSON field";
+        type = "regex_replace";
+        extractor_config = toJSON ({
+          regex = ".*";
+          replacement = "-";
+        });
+        target_field = "json";
+        source_field = "json";
+        cursor_strategy = "copy";
+        condition_type = "none";
+      };
+      nginx_access_reduce_message = {
+        input_id = "\${graylog_input.nginx_access_logs.id}";
+        order = 3;
+        title = "Reduced message to path";
+        type = "regex_replace";
+        extractor_config = toJSON ({
+          regex = ''.*request": "(.*?)".*'';
+          replacement = "$1";
+        });
+        target_field = "message";
+        source_field = "message";
+        cursor_strategy = "copy";
+        condition_type = "none";
+      };
+
+    };
+
     graylog_input_static_fields = {
 
       nginx_access_logs = {