nixos-config/terranix/graylog/nginx.nix

/* # use this nginx configuration
   # to send data to these inputs

   log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
     '"facility": "nginx", '
     '"src_addr": "$remote_addr", '
     '"body_bytes_sent": $body_bytes_sent, '
     '"request_time": $request_time, '
     '"response_status": $status, '
     '"request": "$request", '
     '"request_method": "$request_method", '
     '"host": "$host",'
     '"upstream_cache_status": "$upstream_cache_status",'
     '"upstream_addr": "$upstream_addr",'
     '"http_x_forwarded_for": "$http_x_forwarded_for",'
     '"http_referrer": "$http_referer", '
     '"http_user_agent": "$http_user_agent" }';

   access_log syslog:server=${access_log_input} graylog2_json;
   error_log syslog:server=${error_log_input};
*/

with builtins; {

  resource = {

    graylog_input = {

      nginx_access_logs = {
        title = "nginx access log";
        # https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
        type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
        global = true;
        attributes = toJSON ({
          allow_override_date = true;
          bind_address = "0.0.0.0";
          expand_structured_data = false;
          force_rdns = false;
          number_worker_threads = 4;
          port = 12304;
          recv_buffer_size = 1048576;
          store_full_message = false;
        });
      };

      nginx_error_logs = {
        title = "nginx error log";
        # https://javadoc.io/doc/org.graylog2/graylog2-inputs/latest/index.html
        type = "org.graylog2.inputs.syslog.udp.SyslogUDPInput";
        global = true;
        attributes = toJSON ({
          allow_override_date = true;
          bind_address = "0.0.0.0";
          expand_structured_data = false;
          force_rdns = false;
          number_worker_threads = 4;
          port = 12305;
          recv_buffer_size = 1048576;
          store_full_message = false;
        });
      };

    };

    graylog_extractor = {

      # nginx error
      nginx_error_timestamp = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        order = 0;
        title = "Timestamp";
        type = "regex";
        extractor_config = toJSON ({
          regex_value =
            "^.*:\\s(\\d\\d\\d\\d/\\d\\d/\\d\\d\\s\\d\\d:\\d\\d:\\d\\d)\\s.*$";
        });
        target_field = "timestamp";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "none";
        converters = {
          config = toJSON ({ date_format = "yyyy/MM/dd HH:mm:ss "; });
          type = "date";
        };
      };
      nginx_error_server = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        type = "regex";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "string";
        condition_value = "server";
        extractor_config = toJSON ({ regex_value = "server:\\s(.+?)(,|$)"; });
        order = 1;
        target_field = "server";
        title = "server";
      };
      nginx_error_remote_addr = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        type = "regex";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "string";
        condition_value = "client";
        extractor_config = toJSON ({ regex_value = "client:\\s(.+?)(,|$)"; });
        order = 2;
        target_field = "remote_addr";
        title = "remote_addr/client";
      };
      nginx_error_host = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        type = "regex";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "string";
        condition_value = "host";
        extractor_config = toJSON ({ regex_value = ''host:\s"(.+?)"(,|$)''; });
        order = 3;
        target_field = "host";
        title = "host";
      };
      nginx_error_request_path = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        type = "regex";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "string";
        condition_value = "request";
        extractor_config =
          toJSON ({ regex_value = ''request:\s"(.+?)"(,|$)''; });
        order = 4;
        target_field = "request_path";
        title = "request_path/request";
      };
      nginx_error_request_verb = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        type = "regex";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "string";
        condition_value = "request";
        extractor_config = toJSON ({
          regex_value = ''
            request:\s"(GET|HEAD|POST|PUT|DELETE|TRACE|OPTIONS|CONNECT|PATCH).+"(,|$)'';
        });
        order = 5;
        target_field = "request_verb";
        title = "request_verb";
      };

      # nginx access
      nginx_access_json_from_syslog = {
        input_id = "\${graylog_input.nginx_access_logs.id}";
        title = "Get JSON from syslog message";
        type = "regex";
        cursor_strategy = "copy";
        condition_type = "none";
        source_field = "message";
        target_field = "json";
        order = 0;
        extractor_config = toJSON ({ regex_value = "nginx:\\s+(.*)"; });
      };
      nginx_access_extract_json = {
        input_id = "\${graylog_input.nginx_access_logs.id}";
        title = "Extract JSON fields";
        order = 1;
        source_field = "json";
        type = "json";
        cursor_strategy = "copy";
        condition_type = "none";
        extractor_config = toJSON ({
          flatten = true;
          list_separator = ", ";
          kv_separator = "=";
          key_prefix = "";
          key_separator = "_";
          replace_key_whitespace = false;
          key_whitespace_replacement = "_";
        });
      };
      nginx_access_empty_json = {
        input_id = "\${graylog_input.nginx_access_logs.id}";
        order = 2;
        title = "Empty JSON field";
        type = "regex_replace";
        extractor_config = toJSON ({
          regex = ".*";
          replacement = "-";
        });
        target_field = "json";
        source_field = "json";
        cursor_strategy = "copy";
        condition_type = "none";
      };
      nginx_access_reduce_message = {
        input_id = "\${graylog_input.nginx_access_logs.id}";
        order = 3;
        title = "Reduced message to path";
        type = "regex_replace";
        extractor_config = toJSON ({
          regex = ''.*request": "(.*?)".*'';
          replacement = "$1";
        });
        target_field = "message";
        source_field = "message";
        cursor_strategy = "copy";
        condition_type = "none";
      };

    };

    graylog_input_static_fields = {

      nginx_access_logs = {
        input_id = "\${graylog_input.nginx_access_logs.id}";
        fields = {
          from_nginx = true;
          nginx_error = false;
          nginx_access = true;
        };
      };

      nginx_error_logs = {
        input_id = "\${graylog_input.nginx_error_logs.id}";
        fields = {
          from_nginx = true;
          nginx_error = true;
          nginx_access = false;
        };
      };

    };

    graylog_stream = {
      nginx5xx = {
        title = "nginx 5xx";
        description = "all requests answered with a 5xx response";
        index_set_id = "\${graylog_index_set.default.id}";
        disabled = false;
        matching_type = "AND";
      };
      nginx4xx = {
        title = "nginx 4xx";
        description = "all requests answered with a 4xx response";
        index_set_id = "\${graylog_index_set.default.id}";
        disabled = false;
        matching_type = "AND";
      };
      nginx2xx = {
        title = "nginx 2xx";
        description = "all requests answered with a 2xx response";
        index_set_id = "\${graylog_index_set.default.id}";
        disabled = false;
        matching_type = "AND";
      };
      nginx_access = {
        title = "nginx access";
        description = "all requests";
        index_set_id = "\${graylog_index_set.default.id}";
        disabled = false;
        matching_type = "AND";
      };
      nginx_error = {
        title = "nginx error";
        description = "all errors";
        index_set_id = "\${graylog_index_set.default.id}";
        disabled = false;
        matching_type = "AND";
      };
    };

    graylog_stream_rule = let
      nq_stream_rule = field: value: stream_id: {
        inherit field value stream_id;
        type = 1;
        inverted = true;
      };
      eq_stream_rule = field: value: stream_id: {
        inherit field value stream_id;
        type = 1;
        inverted = false;
      };
      gt_stream_rule = field: value: stream_id: {
        inherit field value stream_id;
        type = 3;
        inverted = false;
      };
      lt_stream_rule = field: value: stream_id: {
        inherit field value stream_id;
        type = 4;
        inverted = false;
      };
      between = min: max: stream_id: {
        "is_nginx_access_${min}_${max}" =
          (eq_stream_rule "nginx_access" true stream_id);
        "nginx_above${min}" = (gt_stream_rule "response_status" min stream_id);
        "nginx_below${max}" = (lt_stream_rule "response_status" max stream_id);
      };
    in (between "499" "600" "\${graylog_stream.nginx5xx.id}")
    // (between "399" "500" "\${graylog_stream.nginx4xx.id}")
    // (between "199" "300" "\${graylog_stream.nginx2xx.id}") // {
      is_nginx_access = (eq_stream_rule "nginx_access" true
        "\${graylog_stream.nginx_access.id}");
      is_nginx_error =
        (eq_stream_rule "nginx_error" true "\${graylog_stream.nginx_error.id}");
    };

  };
}