How can I remove the duplicate in the logs and prevent to create new docs

Elastic Stack Logstash
1

Hello everyone!

I have this logs

{
  "_index": ".ds-my-neoada-stream-2023.09.14-000005",
  "_id": "T8v5sIoB0eBbzdbCLRnW",
  "_version": 1,
  "_score": 0,
  "_source": {
    "from_plant": "N/A",
    "tick_current": "IT-Operations",
    "ticket_xsubj": "PC Assistance",
    "ticket_xstat": "In Progress",
    "ticket_xcompl": "Low",
    "created_at": "2023-09-18T01:50:16.000000Z",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "@timestamp": "2023-09-20T05:00:29.000Z",
    "updated_at": "2023-09-20T05:00:29.000000Z",
    "@version": "1",
    "host": {
      "containerized": false,
      "name": "neoada.acccorp.com.ph",
      "id": "80c3fb03e99b4b8e8901c3d999bed9e7",
      "mac": [
        "00-0C-29-C6-13-EA"
      ],
      "hostname": "neoada.acccorp.com.ph",
      "architecture": "x86_64",
      "ip": [
        "XXXXXX",
        "fe80::88d6:a3ca:cf67:8448",
        "fe80::4325:2c6d:7b52:2089"
      ],
      "os": {
        "name": "CentOS Linux",
        "platform": "centos",
        "version": "7 (Core)",
        "codename": "Core",
        "kernel": "3.10.0-1160.95.1.el7.x86_64",
        "type": "linux",
        "family": "redhat"
      }
    },
    "ticket_xprio": "Urgent",
    "ticket_xagent": "dbteves",
    "tick_teampips": "dbteves",
    "agent": {
      "name": "Neoada-Server",
      "id": "37d5bd3a-55c8-4b17-8934-6544ecee3353",
      "type": "filebeat",
      "version": "8.9.0",
      "ephemeral_id": "a83bd7a2-69b7-48a5-9674-fa38d43da5af"
    },
    "is_viber": "N/A",
    "ticket_xuser": "racatubay",
    "message": "2023-09-20 13:00:29 (192.168.111.60)-IT Help Desk Ticket Action Updated To: {\"ticket_id\":10384,\"ticket_xno\":\"2023-09-18-IT-005\",\"ticket_xuser\":\"racatubay\",\"ticket_xdate\":\"2023-09-18 09:50:16\",\"ticket_xsubj\":\"PC Assistance\",\"ticket_zdesc\":\"PC Assistance\",\"ticket_xcateg\":\"80\",\"ticket_znote\":\"N/A\",\"ticket_xloc\":\"Head Office (Cubao)\",\"ticket_xprio\":\"Urgent\",\"ticket_xagent\":\"dbteves\",\"ticket_xaction\":\"2023-09-20 13:00:29\",\"ticket_xstat\":\"In Progress\",\"tick_account\":\"dbteves\",\"tick_team\":\"N/A\",\"tick_teampips\":\"dbteves\",\"tick_stages\":\"N/A\",\"tick_current\":\"IT-Operations\",\"from_plant\":\"N/A\",\"ticket_xcompl\":\"Low\",\"is_viber\":\"N/A\",\"updated_at\":\"2023-09-20T05:00:29.000000Z\",\"created_at\":\"2023-09-18T01:50:16.000000Z\"}",
    "ticket_zdesc": "PC Assistance",
    "ticket_xcateg": "80",
    "ticket_znote": "N/A",
    "ecs": {
      "version": "8.0.0"
    },
    "ticket_xloc": "Head Office (Cubao)",
    "ticket_xno": "2023-09-18-IT-005",
    "ticket_xaction": "2023-09-20 13:00:29",
    "tick_stages": "N/A",
    "log": {
      "file": {
        "path": "/opt/lampp/htdocs/ada_v2/storage/app/2023-09-20_AT.log"
      },
      "offset": 193287
    },
    "event": {
      "original": "2023-09-20 13:00:29 (192.168.111.60)-IT Help Desk Ticket Action Updated To: {\"ticket_id\":10384,\"ticket_xno\":\"2023-09-18-IT-005\",\"ticket_xuser\":\"racatubay\",\"ticket_xdate\":\"2023-09-18 09:50:16\",\"ticket_xsubj\":\"PC Assistance\",\"ticket_zdesc\":\"PC Assistance\",\"ticket_xcateg\":\"80\",\"ticket_znote\":\"N/A\",\"ticket_xloc\":\"Head Office (Cubao)\",\"ticket_xprio\":\"Urgent\",\"ticket_xagent\":\"dbteves\",\"ticket_xaction\":\"2023-09-20 13:00:29\",\"ticket_xstat\":\"In Progress\",\"tick_account\":\"dbteves\",\"tick_team\":\"N/A\",\"tick_teampips\":\"dbteves\",\"tick_stages\":\"N/A\",\"tick_current\":\"IT-Operations\",\"from_plant\":\"N/A\",\"ticket_xcompl\":\"Low\",\"is_viber\":\"N/A\",\"updated_at\":\"2023-09-20T05:00:29.000000Z\",\"created_at\":\"2023-09-18T01:50:16.000000Z\"}"
    },
    "tick_team": "N/A",
    "ticket_xdate": "2023-09-18 09:50:16",
    "ip": "192.168.111.60",
    "input": {
      "type": "filestream"
    },
    "ticket_id": 10384,
    "tick_account": "dbteves"
  },
  "fields": {
    "ticket_xagent": [
      "dbteves"
    ],
    "tick_stages": [
      "N/A"
    ],
    "ticket_znote": [
      "N/A"
    ],
    "host.hostname": [
      "XXXXXX"
    ],
    "host.mac": [
      "00-0C-29-C6-13-EA"
    ],
    "ticket_xloc": [
      "Head Office (Cubao)"
    ],
    "host.os.version": [
      "7 (Core)"
    ],
    "ticket_xno": [
      "2023-09-18-IT-005"
    ],
    "host.os.name": [
      "CentOS Linux"
    ],
    "agent.name": [
      "Neoada-Server"
    ],
    "host.name": [
      "XXXXXX"
    ],
    "ticket_xcompl": [
      "Low"
    ],
    "event.original": [
      "2023-09-20 13:00:29 (192.168.111.60)-IT Help Desk Ticket Action Updated To: {\"ticket_id\":10384,\"ticket_xno\":\"2023-09-18-IT-005\",\"ticket_xuser\":\"racatubay\",\"ticket_xdate\":\"2023-09-18 09:50:16\",\"ticket_xsubj\":\"PC Assistance\",\"ticket_zdesc\":\"PC Assistance\",\"ticket_xcateg\":\"80\",\"ticket_znote\":\"N/A\",\"ticket_xloc\":\"Head Office (Cubao)\",\"ticket_xprio\":\"Urgent\",\"ticket_xagent\":\"dbteves\",\"ticket_xaction\":\"2023-09-20 13:00:29\",\"ticket_xstat\":\"In Progress\",\"tick_account\":\"dbteves\",\"tick_team\":\"N/A\",\"tick_teampips\":\"dbteves\",\"tick_stages\":\"N/A\",\"tick_current\":\"IT-Operations\",\"from_plant\":\"N/A\",\"ticket_xcompl\":\"Low\",\"is_viber\":\"N/A\",\"updated_at\":\"2023-09-20T05:00:29.000000Z\",\"created_at\":\"2023-09-18T01:50:16.000000Z\"}"
    ],
    "host.os.type": [
      "linux"
    ],
    "tick_teampips": [
      "dbteves"
    ],
    "ip": [
      "192.168.111.60"
    ],
    "input.type": [
      "filestream"
    ],
    "log.offset": [
      193287
    ],
    "agent.hostname": [
      "Neoada-Server"
    ],
    "ticket_xcateg": [
      "80"
    ],
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "is_viber": [
      "N/A"
    ],
    "agent.id": [
      "37d5bd3a-55c8-4b17-8934-6544ecee3353"
    ],
    "ticket_xuser": [
      "racatubay"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.containerized": [
      false
    ],
    "agent.version": [
      "8.9.0"
    ],
    "tick_account": [
      "dbteves"
    ],
    "host.os.family": [
      "redhat"
    ],
    "ticket_xstat": [
      "In Progress"
    ],
    "from_plant": [
      "N/A"
    ],
    "created_at": [
      "2023-09-18T01:50:16.000000Z"
    ],
    "ticket_xsubj": [
      "PC Assistance"
    ],
    "ticket_xprio": [
      "Urgent"
    ],
    "tick_team": [
      "N/A"
    ],
    "host.ip": [
      "XXXXXX",
      "fe80::88d6:a3ca:cf67:8448",
      "fe80::4325:2c6d:7b52:2089"
    ],
    "agent.type": [
      "filebeat"
    ],
    "updated_at": [
      "2023-09-20T05:00:29.000000Z"
    ],
    "ticket_xaction": [
      "2023-09-20 13:00:29"
    ],
    "host.os.kernel": [
      "3.10.0-1160.95.1.el7.x86_64"
    ],
    "@version": [
      "1"
    ],
    "host.id": [
      "80c3fb03e99b4b8e8901c3d999bed9e7"
    ],
    "tick_current": [
      "IT-Operations"
    ],
    "ticket_zdesc": [
      "PC Assistance"
    ],
    "host.os.codename": [
      "Core"
    ],
    "ticket_id": [
      10384
    ],
    "message": [
      "2023-09-20 13:00:29 (192.168.111.60)-IT Help Desk Ticket Action Updated To: {\"ticket_id\":10384,\"ticket_xno\":\"2023-09-18-IT-005\",\"ticket_xuser\":\"racatubay\",\"ticket_xdate\":\"2023-09-18 09:50:16\",\"ticket_xsubj\":\"PC Assistance\",\"ticket_zdesc\":\"PC Assistance\",\"ticket_xcateg\":\"80\",\"ticket_znote\":\"N/A\",\"ticket_xloc\":\"Head Office (Cubao)\",\"ticket_xprio\":\"Urgent\",\"ticket_xagent\":\"dbteves\",\"ticket_xaction\":\"2023-09-20 13:00:29\",\"ticket_xstat\":\"In Progress\",\"tick_account\":\"dbteves\",\"tick_team\":\"N/A\",\"tick_teampips\":\"dbteves\",\"tick_stages\":\"N/A\",\"tick_current\":\"IT-Operations\",\"from_plant\":\"N/A\",\"ticket_xcompl\":\"Low\",\"is_viber\":\"N/A\",\"updated_at\":\"2023-09-20T05:00:29.000000Z\",\"created_at\":\"2023-09-18T01:50:16.000000Z\"}"
    ],
    "ticket_xdate": [
      "2023-09-18 09:50:16"
    ],
    "@timestamp": [
      "2023-09-20T05:00:29.000Z"
    ],
    "host.os.platform": [
      "centos"
    ],
    "log.file.path": [
      "/opt/lampp/htdocs/ada_v2/storage/app/2023-09-20_AT.log"
    ],
    "agent.ephemeral_id": [
      "a83bd7a2-69b7-48a5-9674-fa38d43da5af"
    ]
  }
}

but whenever the fields updated, it creates new document wiith unique id.

How can I prevent duplication and creation of new docs?

This is my logstash conf:

input {
    pipeline {
        address => neoadaLog
    }

} # <--- END OF INPUT --->

filter {

  if [log][file][path] =~ /\/opt\/lampp\/htdocs\/ada_v2\/storage\/app\/.+\.log/ {

    grok {
			match => { "message" => ["%{TIMESTAMP_ISO8601:[@metadata][timestamp]}%{SPACE}\(%{IP:ip}\)-%{DATA:log}: %{GREEDYDATA:[@metadata][logmsg]}"] }
      overwrite => [ "[@metadata][timestamp]" , "[@metadata][logmsg]" ]
	  }

    json { source => "[@metadata][logmsg]" }

    date {
      match => ["[@metadata][timestamp]", "yyyy-MM-dd HH:mm:ss"]
		  timezone => "Asia/Manila"
		  target => "@timestamp"
    }

    
  }
  

} # <--- END OF FILTER --->

output { 
  if [@metadata][pipeline] {
    elasticsearch {
      hosts => ["myIpaddress"]
      user => "myuser"
      password => "mypassword!"
      cacert => "/etc/elasticsearch/certs/http_ca.crt"
      ssl => true
      ssl_certificate_verification => false
      
      
      #INDEX TEMPLATE
      pipeline => "%{[@metadata][pipeline]}"
      template => "/etc/logstash/template/neoada.filebeat.json"
      template_name => "neoada-log"
      index => "my-neoada-stream"
#      template_overwrite => true
      action => "create"
    }
  }
  else {
    elasticsearch {
      hosts => ["myIpaddress"]
      user => "myuser"
      password => "mypassword!"
      cacert => "/etc/elasticsearch/certs/http_ca.crt"
      ssl => true
      ssl_certificate_verification => false

      
      #INDEX TEMPLATE
      template => "/etc/logstash/template/neoada.filebeat.json"
      template_name => "neoada-log"
      index => "my-neoada-stream"
#      template_overwrite => true
      action => "create"
    }
  }

  stdout {
    codec => rubydebug
  }
  file {
    codec => json
    path => "/var/log/logstash/neoada.log-%{+YYYY-MM-dd}.txt"
  }
} # <--- END OF OUTPUT --->
2

Hi @Cruz,
i think you need to add the document_id in the output section and add the document id you want to update. If you dont specify a document id elastic automatically create a new document.

Let me no if this can be a solution

3

Thank you @Samuele_Lolli for your kind response.

I just revised my output elasticsearch.

elasticsearch {
      hosts => ["https://172.16.24.18:9200"]
      user => "myuser"
      password => "mypassword!"
      cacert => "/etc/elasticsearch/certs/http_ca.crt"
      ssl => true
      ssl_certificate_verification => false
      document_id => "%{[@metadata][generated_id]}"
      doc_as_upsert => true 
      action => "update"
    }

Is it necessary to specify the name of the index/indices?

4

is the format correct?

 index => ".ds-my-neoada-stream-%{+yyyy.MM.dd}-%{+000000}"

I want to get my latest indices and I can still get them every rollover.
I am using data stream and this is the name of my indices:

.ds-my-neoada-stream-2023.09.14-000001
.ds-my-neoada-stream-2023.09.14-000002
.ds-my-neoada-stream-2023.09.14-000004
.ds-my-neoada-stream-2023.09.14-000005
5

hi again, i think that solution can work fine
Try it and let me know :slight_smile:

6

its not working it creates new indices
I want to update the .ds-my-neoada-stream-2023-09-14-0005
but when I update a value in my fields it creates new indices which is not what I want to happen.
Do you have any solution for this?

7

If you set the document_id it will make sure the document is unique within the index it is written to. Other documents in other indexes may have the same _id.

8

Data stream are append only, you can't update using Logstash.

You will need to change to normal indices and also no use rollover.