HTTP Poller in Logstash - input separation to multiple fields

Hello,
I have just started my adventure with Logstash.

I would like to create new json in elasticsearch for every new line. By default whole file is loaded in a message field:
Kibana output:

{
  "_index": "blacklists",
  "_type": "default",
  "_id": "pf3k_2QB9sEBYW4CK4AA",
  "_version": 1,
  "_score": null,
  "_source": {
    "@timestamp": "2018-08-03T13:05:00.569Z",
    "tags": [
      "_jsonparsefailure",
      "c2_info",
      "ipaddress"
    ],
    "@version": "1",
    "message": "#############################################################\n## Master Feed of known, active and non-sinkholed C&Cs IP \n## addresses\n## \n## HIGH-CONFIDENCE FAMILIES ONLY\n## \n## Feed generated at: 2018-08-03 12:13 \n##\n## Feed Provided By: John Bambenek of Bambenek Consulting\n## jcb@bambenekconsulting.com // http://bambenekconsulting.com\n## Use of this feed is governed by the license here: \n## http://osint.bambenekconsulting.com/license.txt,
    "client": "204.11.56.48",
    "http_poller_metadata": {
      "name": "bembenek_c2",
      "host": "node1",
      "request": {
        "method": "get",
        "url": "http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt"
      },
      "response_message": "OK",
      "runtime_seconds": 0.27404,
      "response_headers": {
        "content-type": "text/plain",
        "accept-ranges": "bytes",
        "cf-ray": "4448fe69e02197ce-FRA",
        "date": "Fri, 03 Aug 2018 13:05:05 GMT",
        "connection": "keep-alive",
        "last-modified": "Fri, 03 Aug 2018 12:13:44 GMT",
        "server": "cloudflare",
        "vary": "Accept-Encoding",
        "etag": "\"4bac-57286dbe759e4-gzip\""
      },
      "code": 200,
      "times_retried": 0
    }
  },
  "fields": {
    "@timestamp": [
      "2018-08-03T13:05:00.569Z"
    ]
  },
  "sort": [
    1533301500569
  ]
}

is there a way to split the input from the downloaded files, for example from bembenek_c2, to be visible in Kibana like below?

aakamen.com,Domain used by banjori,2018-08-03 12:03,http://osint.bambenekconsulting.com/manual/banjori.txt

Logstash config:

input {
  http_poller {
    urls => {
      bembenek_c2 => "http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt"
      bembenek_c2dom => "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist-high.txt"
      blocklists_all => "http://lists.blocklist.de/lists/all.txt"
    }
    request_timeout => 30
    codec => "json"
    tags => c2_info
    schedule => { cron => "*/10 * * * *"}
    metadata_target => "http_poller_metadata"
  }
}

filter {
        grok {
                match => { "message" => [
                                "%{IPV4:ipaddress}" }
                add_tag => [ "ipaddress" ]
        }
}

output {
 stdout { codec => dots }
    elasticsearch {
        hosts =>  ["10.0.50.51:9200"]
        index => "blacklists"
        document_type => "default"
        template_overwrite => true
    }
   file {
        path           => "/tmp/blacklists.json"
        codec          => json {}
    }
}

Thanks for the hints!

Use a split filter.

filter {
split {
terminator => "\n"
}
}

Do you mind to throw more hint how to use in this use case?

That should work. In fact, since \n is the default terminator just split { } should do. The field option defaults to the message field which also is the field that you want to split.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.