Import as nested type

Elastic Stack Logstash
1

Hello,
I'm trying to import parent-child data.
Here is my config file:

input{
    jdbc{
        jdbc_driver_library => "/usr/library/postgresql-42.7.4.jar"
        jdbc_connection_string => "jdbc:postgresql://185.**.**.28:5432/feed"
        jdbc_user => "airflow"
        jdbc_password => "**"
        jdbc_driver_class => "org.postgresql.Driver"
        statement => "select distinct source from public.news"
    }
}
filter{
jdbc_streaming {
jdbc_driver_library => "/usr/library/postgresql-42.7.4.jar"
jdbc_connection_string => "jdbc:postgresql://185.**.**.28:5432/feed"
jdbc_user => "airflow"
jdbc_password => "**"
jdbc_driver_class => "org.postgresql.Driver"
parameters => {"source" => "source"}
statement => "select title,article, url, views from public.news where source = :source limit 10"
target => "posts"

}
mutate{
    remove_field => ["@version","@timestamp"]
}
}
output{
    elasticsearch{
        hosts => ["https://172.21.0.2:9200"]
        ssl_certificate_authorities => '/usr/library/http_ca.crt'
        ssl_verification_mode => "full"
        user => "elastic"
        password => "*****"
        index => "content2"
        document_id => "%{source}"
    }
    stdout {
        codec => rubydebug
    }
}

and here is the index mapping which has been created automatically:


{
  "mappings": {
    "properties": {
      "posts": {
        "properties": {
          "article": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "title": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "url": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "views": {
            "type": "long"
          }
        }
      },
      "source": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
}

Questions:

  1. How to assign to posts property nested type
  2. How to make posts subattribute of source?

Any help is greatly appreciated! Thank you

2

Ok, as usual there were no answers.
The solution is a template in output part. Just define a template and voila.

{
  "index_patterns" : ["content*"],
  "priority" : 1,
  "template": {
    "settings" : {
        "number_of_shards" : "1",
        "number_of_replicas" : "0",
        "refresh_interval": "10s"
    },
    "mappings": {
    "properties": {
      "posts": {
        "type": "nested",
        "include_in_parent": true,
        "properties": {
          "article": {
            "type":     "text",
            "fielddata": true,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "title": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "url": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "views": {
            "type": "long"
          }
        }
      },
      "source": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
  }
}

Bad news is that Kibana dashboard doesn't support nested type. I wish i knew it 3 days ago..