So I'm in the process of moving these logs to our elastic cloud and our on-prem version of the conf file works. I will put that below as well.
1,2,5981253,35527805,1637596724,2,35527805,95174559,"819133757527","",0,16,4,111951775,11782,4,20,0,0,0,0,0,0,"0","0",35527821,2,0,1178535946,"5280","5280","xxx",0,0,4,1178535946,24584,4,20,0,0,0,0,0,0,"0","0",1637596724,1637597207,"1871","17b1fafc-c36f-4cbd-b1e1-b00c21adc633","Enterprise DN","","Enterprise DN","ICDPort",483,"GC_SIP_trunk","SEP94E6F7EA5E01",12,10,0,10,0,4,0,"xxxx",10,"","",0,"",2,3,0,0,64,64,"","","","xxx","xxx",0,0,0,0,0,0,0,0,0,0,0,0,1,"EDE8BE80000100000004FC453D0E080A",3,"00000000005B4445021E1C8D00000000",0,0,0,"xxx","1755",2,"","","","","","","","",0,0,"","email","email","","","","","",0,0,0,"5280","5280","1871","1755"
input {
file {
path =>"/home/cucmhome/september/cdr*"
type => "cucm-cdr"
add_field => { "cucm_clustername" => "CUCM_Cluster_1" }
}
file {
path =>"/home/cucmhome/september/cmr*"
type => "cucm-cmr"
add_field => { "cucm_clustername" => "CUCM_Cluster_1" }
}
}
filter {
if [type] == "cucm-cdr"{
if ([message] =~ /^\"cdrRecordType/) {
drop{}
}
if ([message] =~ /^INTEGER/) {
drop{}
}
csv {
columns => ["cdrRecordType","globalCallID_callManagerId","globalCallID_callId","origLegCallIdentifier","dateTimeOrigination","origNodeId","origSpan","origIpAddr","callingPartyNumber","callingPartyUnicodeLoginUserID","origCause_location","origCause_value","origPrecedenceLevel","origMediaTransportAddress_IP","origMediaTransportAddress_Port","origMediaCap_payloadCapability","origMediaCap_maxFramesPerPacket","origMediaCap_g723BitRate","origVideoCap_Codec","origVideoCap_Bandwidth","origVideoCap_Resolution","origVideoTransportAddress_IP","origVideoTransportAddress_Port","origRSVPAudioStat","origRSVPVideoStat","destLegIdentifier","destNodeId","destSpan","destIpAddr","originalCalledPartyNumber","finalCalledPartyNumber","finalCalledPartyUnicodeLoginUserID","destCause_location","destCause_value","destPrecedenceLevel","destMediaTransportAddress_IP","destMediaTransportAddress_Port","destMediaCap_payloadCapability","destMediaCap_maxFramesPerPacket","destMediaCap_g723BitRate","destVideoCap_Codec","destVideoCap_Bandwidth","destVideoCap_Resolution","destVideoTransportAddress_IP","destVideoTransportAddress_Port","destRSVPAudioStat","destRSVPVideoStat","dateTimeConnect","dateTimeDisconnect","lastRedirectDn","pkid","originalCalledPartyNumberPartition","callingPartyNumberPartition","finalCalledPartyNumberPartition","lastRedirectDnPartition","duration","origDeviceName","destDeviceName","origCallTerminationOnBehalfOf","destCallTerminationOnBehalfOf","origCalledPartyRedirectOnBehalfOf","lastRedirectRedirectOnBehalfOf","origCalledPartyRedirectReason","lastRedirectRedirectReason","destConversationId","globalCallId_ClusterID","joinOnBehalfOf","comment","authCodeDescription","authorizationLevel","clientMatterCode","origDTMFMethod","destDTMFMethod","callSecuredStatus","origConversationId","origMediaCap_Bandwidth","destMediaCap_Bandwidth","authorizationCodeValue","outpulsedCallingPartyNumber","outpulsedCalledPartyNumber","origIpv4v6Addr","destIpv4v6Addr","origVideoCap_Codec_Channel2","origVideoCap_Bandwidth_Channel2","origVideoCap_Resolution_Channel2","origVideoTransportAddress_IP_Channel2","origVideoTransportAddress_Port_Channel2","origVideoChannel_Role_Channel2","destVideoCap_Codec_Channel2","destVideoCap_Bandwidth_Channel2","destVideoCap_Resolution_Channel2","destVideoTransportAddress_IP_Channel2","destVideoTransportAddress_Port_Channel2","destVideoChannel_Role_Channel2","IncomingProtocolID","IncomingProtocolCallRef","OutgoingProtocolID","OutgoingProtocolCallRef","currentRoutingReason","origRoutingReason","lastRedirectingRoutingReason","huntPilotPartition","huntPilotDN","calledPartyPatternUsage","IncomingICID","IncomingOrigIOI","IncomingTermIOI","OutgoingICID","OutgoingOrigIOI","OutgoingTermIOI","outpulsedOriginalCalledPartyNumber","outpulsedLastRedirectingNumber","wasCallQueued","totalWaitTimeInQueue","callingPartyNumber_uri","originalCalledPartyNumber_uri","finalCalledPartyNumber_uri","lastRedirectDn_uri"]
separator => ","
}
if "_csvparsefailure" in [tags] {
drop{}
}
if "_dateparsefailure" in [tags] {
drop{}
}
mutate {
convert => [ "duration", "integer" ]
}
mutate {
add_field => ["dateTimeImport", "%{@timestamp}"]
}
date {
match => ["dateTimeOrigination", "UNIX"]
}
date {
match => ["dateTimeOrigination", "UNIX"]
target => "dateTimeOrigination_formatted"
}
date {
match => ["dateTimeConnect", "UNIX"]
target => "dateTimeConnect_formatted"
}
date {
match => ["dateTimeDisconnect", "UNIX"]
target => "dateTimeDisconnect_formatted"
}
translate {
field => "destMediaCap_payloadCapability"
destination => "destMediaCap_payloadCapability_text"
dictionary => [
"0","NoCodec",
"1","NonStandard",
"2","G711Alaw 64k",
"3","G711Alaw 56k",
"4","G711mu-law 64k",
"5","G711mu-law 56k",
"6","G722 64k",
"7","G722 56k",
"8","G722 48k",
"9","G7231",
"10","G728",
"11","G729",
"12","G729AnnexA",
"13","Is11172AudioCap",
"14","Is13818AudioCap",
"15","G.729AnnexB",
"16","G.729 Annex AwAnnexB",
"18","GSM Full Rate",
"19","GSM Half Rate",
"20","GSM Enhanced Full Rate",
"25","Wideband 256K",
"32","Data 64k",
"33","Data 56k",
"40","G7221 32K",
"41","G7221 24K",
"42","AAC",
"80","GSM",
"81","ActiveVoice",
"82","G726_32K",
"83","G726_24K",
"84","G726_16K",
"86","iLBC",
"89","iSAC",
"100","H261",
"101","H263",
"102","Vieo",
"103","H264",
"106","H224"
]
}
}
if [type] == "cucm-cmr"{
if ([message] =~ /^\"cdrRecordType/) {
drop{}
}
if ([message] =~ /^INTEGER/) {
drop{}
}
csv {
columns => ["cdrRecordType","globalCallID_callManagerId","globalCallID_callId","nodeId","directoryNum","callIdentifier","dateTimeStamp","numberPacketsSent","numberOctetsSent","numberPacketsReceived","numberOctetsReceived","numberPacketsLost","jitter","latency","pkid","directoryNumPartition","globalCallId_ClusterID","deviceName","varVQMetrics"]
separator => ","
}
if "_csvparsefailure" in [tags] {
drop{}
}
if "_dateparsefailure" in [tags] {
drop{}
}
kv {
source => "varVQMetrics"
field_split => ";"
prefix => "VQMetrics_"
}
mutate {
add_field => ["dateTimeImport", "%{@timestamp}"]
}
date {
match => ["dateTimeStamp", "UNIX"]
}
date {
match => ["dateTimeStamp", "UNIX"]
target => "dateTimeStamp_formatted"
}
}
}
output {
elasticsearch {
# index => "%{type}-%{+dd.MM.YYYY}"
index => "cucm_cdr"
hosts => ["localhost:9200"]
user => ""
password => ""
}
}