HTTP Poller in Logstash - input separation to multiple fields

Elastic Stack Logstash
1

Hello,
I have just started my adventure with Logstash.

I would like to create new json in elasticsearch for every new line. By default whole file is loaded in a message field:
Kibana output:

{
  "_index": "blacklists",
  "_type": "default",
  "_id": "pf3k_2QB9sEBYW4CK4AA",
  "_version": 1,
  "_score": null,
  "_source": {
    "@timestamp": "2018-08-03T13:05:00.569Z",
    "tags": [
      "_jsonparsefailure",
      "c2_info",
      "ipaddress"
    ],
    "@version": "1",
    "message": "#############################################################\n## Master Feed of known, active and non-sinkholed C&Cs IP \n## addresses\n## \n## HIGH-CONFIDENCE FAMILIES ONLY\n## \n## Feed generated at: 2018-08-03 12:13 \n##\n## Feed Provided By: John Bambenek of Bambenek Consulting\n## jcb@bambenekconsulting.com // http://bambenekconsulting.com\n## Use of this feed is governed by the license here: \n## http://osint.bambenekconsulting.com/license.txt,
    "client": "204.11.56.48",
    "http_poller_metadata": {
      "name": "bembenek_c2",
      "host": "node1",
      "request": {
        "method": "get",
        "url": "http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt"
      },
      "response_message": "OK",
      "runtime_seconds": 0.27404,
      "response_headers": {
        "content-type": "text/plain",
        "accept-ranges": "bytes",
        "cf-ray": "4448fe69e02197ce-FRA",
        "date": "Fri, 03 Aug 2018 13:05:05 GMT",
        "connection": "keep-alive",
        "last-modified": "Fri, 03 Aug 2018 12:13:44 GMT",
        "server": "cloudflare",
        "vary": "Accept-Encoding",
        "etag": "\"4bac-57286dbe759e4-gzip\""
      },
      "code": 200,
      "times_retried": 0
    }
  },
  "fields": {
    "@timestamp": [
      "2018-08-03T13:05:00.569Z"
    ]
  },
  "sort": [
    1533301500569
  ]
}

is there a way to split the input from the downloaded files, for example from bembenek_c2, to be visible in Kibana like below?

aakamen.com,Domain used by banjori,2018-08-03 12:03,http://osint.bambenekconsulting.com/manual/banjori.txt

Logstash config:

input {
  http_poller {
    urls => {
      bembenek_c2 => "http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt"
      bembenek_c2dom => "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist-high.txt"
      blocklists_all => "http://lists.blocklist.de/lists/all.txt"
    }
    request_timeout => 30
    codec => "json"
    tags => c2_info
    schedule => { cron => "*/10 * * * *"}
    metadata_target => "http_poller_metadata"
  }
}

filter {
        grok {
                match => { "message" => [
                                "%{IPV4:ipaddress}" }
                add_tag => [ "ipaddress" ]
        }
}

output {
 stdout { codec => dots }
    elasticsearch {
        hosts =>  ["10.0.50.51:9200"]
        index => "blacklists"
        document_type => "default"
        template_overwrite => true
    }
   file {
        path           => "/tmp/blacklists.json"
        codec          => json {}
    }
}

Thanks for the hints!

2

Use a split filter.

3

filter {
split {
terminator => "\n"
}
}

Do you mind to throw more hint how to use in this use case?

4

That should work. In fact, since \n is the default terminator just split { } should do. The field option defaults to the message field which also is the field that you want to split.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.