Ngnix log to Elasticsearch

nginx-es.conf

 

input {
 file {
        path => "/opt/logtest/nginx_access.log.1"
        start_position => "beginning"
       sincedb_path => "/opt/logstash-2.3.4/sincedb/"
    }

}

filter {
   ruby{
     init => "@kname=['remote_addr','time_local','http_host','request','status','body_bytes_sent','http_referer','http_user_agent','upstream_response_time','request_time']"
     code => "event.append(Hash[@kname.zip(event['message'].split(' | '))])"
   }
  if [http_user_agent] =="-" {
     drop { }
  }
  if [request] {
    ruby {
      init => "@kname = ['method','uri','verb']"
      code => "event.append(Hash[@kname.zip(event['request'].split(' '))])"
    }
    if [uri] {
       ruby {
         init => "@kname = ['url_path', 'url_args']"
         code => "event.append(Hash[@kname.zip(event['uri'].split('?'))])"
      }
    }
  }
  geoip {
    source => "remote_addr"
  }
  mutate {
     convert => [
          "body_bytes_sent", "integer",
          "status", "integer",
          "upstream_response_time", "float",
          "request_time", "float"
        ]
   add_field => {"type" => "nginx"}
   remove_field => ["uri","request"]
  }
  date {
    match => ["time_local", "dd/MMM/yyyy:HH:mm:ss +0800", "ISO8601"]
    target => "@timestamp"
    remove_field => ["time_local", "message"]
  }

}

  

output {
  stdout{ codec => rubydebug}

  elasticsearch {
     hosts => ["192.168.0.135:9200"]
     index => "logstash-%{type}-%{+YYYY.MM.dd}"
     workers => 2
     template_overwrite => true
   }
}

 

log content sample

192.168.100.10 | 29/Jul/2016:17:19:36 +0800 | api2.unichat.cn | GET / HTTP/1.1 | 200 | 17 | - | - | 0.052 | 0.052
118.212.135.122 | 29/Jul/2016:17:19:44 +0800 | api2.unichat.cn | GET /api/uchat?page=2&recommend_sort=1469783728640 HTTP/1.1 | 200 | 3499 | - | PhotoFlow/1.0.9 (iPhone; iOS 9.2.1; Scale/2.00) | 0.200 | 0.201

 

log_format uchat "$remote_addr | $time_local | $http_host | $request | $status | $body_bytes_sent | $http_referer | $http_user_agent | $upstream_response_time | $request_time";

 

output

 

{
                  "@version" => "1",
                "@timestamp" => "2016-08-03T02:18:21.000Z",
                      "path" => "/opt/logtest/nginx_access.log",
                      "host" => "inok-c0",
               "remote_addr" => "125.71.215.46",
                 "http_host" => "api2.unichat.cn",
                    "status" => 200,
           "body_bytes_sent" => 17,
              "http_referer" => "-",
           "http_user_agent" => "check_http/v1.4.16 (nagios-plugins 1.4.16)",
    "upstream_response_time" => 0.042,
              "request_time" => 0.043,
                    "method" => "GET",
                      "verb" => "HTTP/1.1",
                  "url_path" => "/",
                  "url_args" => nil,
                     "geoip" => {
                      "ip" => "125.71.215.46",
           "country_code2" => "CN",
           "country_code3" => "CHN",
            "country_name" => "China",
          "continent_code" => "AS",
             "region_name" => "32",
               "city_name" => "Chengdu",
                "latitude" => 30.66669999999999,
               "longitude" => 104.06670000000003,
                "timezone" => "Asia/Chongqing",
        "real_region_name" => "Sichuan",
                "location" => [
            [0] 104.06670000000003,
            [1] 30.66669999999999
        ]
    },
                      "type" => "nginx"
}