My setup:
Azure NSG Flowlogs -> Azure Storage -> Logstash with Azure Plugin -> Graylog
As far as I can tell the Azure Logstash plugin is pulling all the logs however I think my filter is then causing some of these to be dropped.
Below is a node stat report and while i would expect a split to increase the number of logs I would not expect the other filters to reduce in number (IN/OUT by 30%)? Is this a bad assumption?
indent preformatted text by 4 spaces
}[root@USA1042LV000914 scalr]# curl -XGET 'localhost:9600/_node/stats/pipelines/main?pretty'
{
"host" : "USA1042LV000914",
"version" : "6.1.3",
"http_address" : "127.0.0.1:9600",
"id" : "0f2a3d5d-7238-4446-93c7-7ae7c061c387",
"name" : "USA1042LV000914",
"pipelines" : {
"main" : {
"events" : {
"duration_in_millis" : 298425,
"in" : 14491,
"out" : 174414,
"filtered" : 174414,
"queue_push_duration_in_millis" : 117
},
"plugins" : {
"inputs" : [ {
"id" : "4c6a0d903c9c9e07d122e2f9d050e2206b14cf121e353f9cc342b704415108e6",
"events" : {
"out" : 2070,
"queue_push_duration_in_millis" : 0
},
"name" : "azureblob"
}, {
"id" : "c85cd2c4eb1fcd60caa66ed5b0703fb6230af02f35f4d5fd7d04033eaab1222b",
"events" : {
"out" : 5693,
"queue_push_duration_in_millis" : 54
},
"name" : "azureblob"
}, {
"id" : "0e01877204de964a6ccded9fcdd03226f02a64ca08605954e9d384bff33a1dd0",
"events" : {
"out" : 6728,
"queue_push_duration_in_millis" : 63
},
"name" : "azureblob"
} ],
"filters" : [ {
"id" : "c75e1e7bc2a261822d83a6311d966a4ecaaf012e9c8bb1ab4ec68207d7ed0aa5",
"events" : {
"duration_in_millis" : 5626,
"in" : 31892,
"out" : 23816
},
"name" : "split"
}, {
"id" : "2068f62b66638423cca18067192b6e579e971ea340f492771c1e98d210efd63f",
"events" : {
"duration_in_millis" : 24318,
"in" : 47632,
"out" : 159923
},
"name" : "split"
}, {
"id" : "aafa4b6167620c4ce9ff08d9d4e3ac312990a8d9596b99444828a5efdc06eb2e",
"events" : {
"duration_in_millis" : 3961,
"in" : 41217,
"out" : 23816
},
"name" : "split"
}, {
"id" : "b5a57f20740ce51a15f2b652a15525c0fd6946bcc560c05ac290b3b868bef371",
"events" : {
"duration_in_millis" : 6818,
"in" : 14491,
"out" : 17401
},
"name" : "split"
}, {
"id" : "f76b29a531f68782bab9dd5a13b8cf225f0a40a27a831704058ae0c0e3e01bfd",
"events" : {
"duration_in_millis" : 186361,
"in" : 183739,
"out" : 159923
},
"name" : "mutate"
}, {
"id" : "d08a08b60bec9f042df0a62d62e4ea560398dd1c4caebbaeccf846fcbeee692f",
"events" : {
"duration_in_millis" : 1567,
"in" : 159923,
"out" : 159923
},
"matches" : 159923,
"name" : "date"
} ],
"outputs" : [ {
"id" : "1c6ce69269f9f1ef0dddfb6e443312586d6d8878ba532b99cd8042321d51b706",
"events" : {
"duration_in_millis" : 7122,
"in" : 159923,
"out" : 159923
},
"name" : "udp"
} ]
},
"reloads" : {
"last_error" : null,
"successes" : 0,
"last_success_timestamp" : null,
"last_failure_timestamp" : null,
"failures" : 0
},
"queue" : {
"type" : "memory"
}
}
}