Logstash only adding one document from mongodb to elasticsearch

[tahaimt]

I am trying to index my mongodb collections in elasticsearch with logstash and to verify that the data is indexed in elasticsearch I am using kibana. Pasted below are my docker-compose and logstash.conf files. After up-ing the stack, I manually go into the mongodb and create a database Files and a collection inside called fileCollection. In the collection I create two documents (pasted in the end). I also use curl -XPUT http://localhost:9200/filesindex to create a new index in elastic search.

After doing the above, I look at the logs of logstash container and every 5 seconds I see the lines below, which makes me think that logstash found some records in mongodb and pushed it to elasticsearch as I see no errors in elasticsearch as well. However, when I goto index management in Kibana, I see that the docs count for the index is only 1 when there are 2 documents in that collection in mongodb.

LOGS
[2020-02-10T16:31:55.644633 #101] DEBUG -- : MONGODB | [468] mongo:27017 #1 | Files.find | STARTED | {"find"=>"fileCollection", "filter"=>{"_id"=>{"$gt"=>BSON::ObjectId('5e4173bffa71070006efc418')}}, "limit"=>10, "lsid"=>{"id"=><BSON::Binary:0x2048 type=uuid data=0xa5ac8cf240ad4974...>}}
D, [2020-02-10T16:31:55.649544 #101] DEBUG -- : MONGODB | [468] mongo:27017 | Files.find | SUCCEEDED | 0.004s
D, [2020-02-10T16:31:55.669449 #101] DEBUG -- : MONGODB | [469] mongo:27017 #1 | Files.listCollections | STARTED | {"listCollections"=>1, "cursor"=>{}, "nameOnly"=>true, "lsid"=>{"id"=><BSON::Binary:0x2048 type=uuid data=0xa5ac8cf240ad4974...>}}
D, [2020-02-10T16:31:55.671403 #101] DEBUG -- : MONGODB | [469] mongo:27017 | Files.listCollections | SUCCEEDED | 0.001s

docker-compose.yml

# Use root/example as user/password credentials
version: '3.1'

services:

  mongo:
    image: mongo
    container_name: mongo
    restart: always
    ports:
      - 27017:27017
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: example
    volumes:
      - ~/Documents/Logstash/mongodata:/data/db

  mongo-express:
    image: mongo-express
    restart: always
    ports:
      - 8081:8081
    environment:
      ME_CONFIG_MONGODB_ADMINUSERNAME: root
      ME_CONFIG_MONGODB_ADMINPASSWORD: example

  elastic:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    container_name: elasticsearch
    restart: always
    expose:
      - 9200
      - 9300
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      discovery.type: single-node
    volumes:
      - ~/Documents/Logstash/esdata:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:7.5.2
    command: bash -c "cd bin && logstash-plugin install logstash-input-mongodb && logstash-plugin install logstash-output-elasticsearch && logstash -f /usr/share/logstash/config/logstash.conf"
    volumes:
      - ~/Documents/Logstash/logstash.conf:/usr/share/logstash/config/logstash.conf
    depends_on:
      - elastic
      - mongo
  kibana:
    image: docker.elastic.co/kibana/kibana:7.5.2
    environment:
      SERVER_NAME: kibana.example.org
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    ports:
      - 5601:5601

logstash.conf

input {
  mongodb {
    uri => "mongodb://root:example@mongo:27017/Files?authSource=admin"
    placeholder_db_dir => "/opt/logstash/"
    placeholder_db_name => "logstash_sqlite.db"
    collection => "fileCollection"
    batch_size => 200
  }
}

filter {
  mutate {
    rename => { "_id" => "mongo_id" }
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    user => "elastic"
    password => "password"
    action => "index"
    index => "filesindex"
	document_type => "document_type"
	document_id => "%{id}" 
  }
}

Documents in http://localhost:8081/db/Files/fileCollection/

{
    _id: ObjectId('5e4173bffa71070006efc418'),
    meta: {
        test: {
            code: 'TESTCODE-123',
            description: 'TEST CODE DESCRIPTION 123',
            name: 'TEST CODE USER FRIENDLY NAME 123'
        },
        tags: [
            'TESTCODE-123',
            'TEST CODE DESCRIPTION 123',
            'TEST CODE USER FRIENDLY NAME 123'
        ],
        fileType: 'pdf',
        uploadedBy: 'system',
        uploadedDate: ISODate('2020-02-10T15:16:15.517Z'),
        fileReference: 'C:/Documents/123.pdf'
    }
}


{
    id: ObjectId(),
    meta: {
        test: {
            code: 'TESTCODE-456',
            description: 'TEST CODE DESCRIPTION 456',
            name: 'TEST CODE USER FRIENDLY NAME 456'
        },
        tags: [
            'TESTCODE-456',
            'TEST CODE DESCRIPTION 456',
            'TEST CODE USER FRIENDLY NAME 456'
        ],
        fileType: 'pdf',
        uploadedBy: 'system',
        uploadedDate: new Date(),
        fileReference: 'C:/Documents/456.pdf'
    }
}