Hi Friends,
i have a long log and i'm trying to parse it with javascript this is the filebeat config that i have:
filebeat.inputs:
- type: log
enabled: true
paths:
- /root/logtest/test.log
# clean_removed: true
# clean_inactive: true
# close_inactive: true
# close_removed: true
multiline.type: pattern
multiline.pattern: '^[[:space:]]'
multiline.negate: false
multiline.match: after
scan_frequency: 1s
processors:
- script:
lang: javascript
id: payment_process
source: >
function process(event) {
var mystr = event.Get("message");
var extractBeginningRegexp = /[a-zA-Z]+:\s*?\[[-][0-9]+\s*?\]\s*?\[\]\s*?\s*?\[([0-9]{4}[-][0-9]{2}-[0-9]{2}\s*?[0-9]{2}:[0-9]{2}:[0-9]{2}),[0-9]+\]\s*?([a-zA-Z]+)?\s*{.+?}\s*?-\s*?([^\s].*)/;
var parseList = mystr.match(extractBeginningRegexp);
var date_ = parseList[1];
var level_ = parseList[2];
var wholeBody = parseList[3].replace(/\s/g, '');
var headers = wholeBody.match(/([hH]eaders=[[{].+?[}\]]+),/)[1];
var headers_ = headers.split('=')[1];
var payload = wholeBody.match(/([pP]ayload=[[{].+?[}\]]+),/)[1];
var payload_ = payload.split('=')[1];
var filter1 = wholeBody.replace(headers + ',' , '').replace(payload + ',' , '');
var To = filter1.match(/(To:[^,]+?),/)[1];
var To_ = To.split(':')[1];
var messageId = filter1.match(/(MessageID:[^,]+?),/)[1];
var messageId_ = messageId.replace('MessageID:', '');
var direction = filter1.match(/(Direction:[^,]+?),/)[1];
var direction_ = direction.split(':')[1];
var filter2 = filter1.replace(To + ',', '').replace(messageId + ',', '').replace(direction + ',', '');
var finalJson = {
date: date_,
level: level_,
headers: headers_,
payload: payload_,
To: To_,
messageId: messageId_,
direction: direction_,
};
var filter2List = filter2.split(',');
for (var i = 0 ; i < filter2List.length; i++) {
var item = filter2List[i];
var arr = item.split('=');
finalJson[arr[0]] = arr[1];
}
event.Put("log", finalJson);
return event;
}
# ============================== Filebeat modules ==============================
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
# ======================= Elasticsearch template setting =======================
setup.ilm.enabled: false
setup.template.name: 'filebeat'
setup.template.pattern: 'filebeat'
setup.template.overwrite: false
setup.template.settings:
index.number_of_shards: 1
index.number_of_replicas: 1
#index.codec: best_compression
#_source.enabled: false
# ================================== General ===================================
#queue.disk:
# max_size: 1GB
# ================================= Dashboards =================================
# =================================== Kibana ===================================
# =============================== Elastic Cloud ================================
# ================================== Outputs ===================================
# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
hosts: ["x.x.x.x:x"]
protocol: "http"
loadbalance: true
# worker: 1
index: "test"
# username: ""
# password: ""
# ------------------------------ Logstash Output -------------------------------
# ================================= Processors =================================
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
# ================================== Logging ===================================
logging.level: warning
logging.to_files: true
logging.files:
path: /root/filebeat/log
name: filebeat
keepfiles: 7
permissions: 0644
# ============================= X-Pack Monitoring ==============================
# ================================= Migration ==================================
and my logs are somehow looks like this :
TID: [-1] [] [2022-01-03 13:43:54,059] INFO {org.apache.synapse.mediators.builtin.LogMediator} - To: , MessageID: urn:uuid:8fb6dfd8-43dc-4a3a-8fdc-8d63b4faeff0, Direction: response, LogName = Fault, RequestID = null, LoadBalancerIP = null, RemoteIP = null, UserID = null, Username = null, ClientID = null, Method = null, Context = null, Resource = null, OutgoingMethod = null, OutgoingContext = null, HttpStatus = 524, EsbErrorCode = 101504, EsbErrorMessage = Send timeout, ProviderResponseTime = NaN, Headers = {"Authorization":"","Content-Type":"application/x-www-form-urlencoded","a":0}, Payload = {"status":{"code":524,"message":"Gateway was able to complete a connection to the origin server, but did not receive a timely HTTP response!"},"meta":{"transactionId":""},"result":{"data":null, "status":{"code":524,"message":"Gateway was able to complete a connection to the origin server, but did not receive a timely HTTP response!"}}}, ApiName = null, ApiVersion = null, ApiBaseContext = null, ApiContext = , ApiResource = null
TID: [-1] [] [2022-01-03 13:43:54,061] INFO {org.apache.synapse.mediators.builtin.LogMediator} - To: , MessageID: urn:uuid:8fb6dfd8-43dc-4a3a-8fdc-8d63b4faeff0, Direction: response, LogName = Respond, RequestID = null, LoadBalancerIP = null, RemoteIP = null, UserID = null, Username = null, ClientID = null, Method = null, Context = null, Resource = , HttpStatus = 524, ResponseTime = NaN, Headers = {"Authorization":"","Content-Type":"application/x-www-form-urlencoded","a":0}, Payload = {"status":{"code":524,"message":"Gateway was able to complete a connection to the origin server, but did not receive a timely HTTP response!"},"meta":{"transactionId":""},"result":{"data":null, "status":{"code":524,"message":"Gateway was able to complete a connection to the origin server, but did not receive a timely HTTP response!"}}}, ApiName = null, ApiVersion = null, ApiBaseContext = null, ApiContext = , ApiResource = null, Description = null
TID: [-1] [] [2022-01-03 13:43:54,062] ERROR {org.apache.synapse.core.axis2.Axis2Sender} - Unexpected error sending message back org.apache.axis2.AxisFault: Unable to determine wsa:Action for outbound message
at org.apache.axis2.handlers.addressing.AddressingOutHandler$WSAHeaderWriter.processWSAAction(AddressingOutHandler.java:311)
at org.apache.axis2.handlers.addressing.AddressingOutHandler$WSAHeaderWriter.writeHeaders(AddressingOutHandler.java:228)
at org.apache.axis2.handlers.addressing.AddressingOutHandler.doInvoke(AddressingOutHandler.java:134)
at org.apache.axis2.handlers.AbstractTemplatedHandler.invoke(AbstractTemplatedHandler.java:44)
at org.apache.axis2.engine.Phase.invokeHandler(Phase.java:340)
at org.apache.axis2.engine.Phase.invoke(Phase.java:313)
at org.apache.axis2.engine.AxisEngine.invoke(AxisEngine.java:261)
at org.apache.axis2.engine.AxisEngine.send(AxisEngine.java:426)
at org.apache.synapse.core.axis2.Axis2Sender.sendBack(Axis2Sender.java:220)
at org.apache.synapse.mediators.builtin.RespondMediator.mediate(RespondMediator.java:46)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:101)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:71)
at org.apache.synapse.mediators.base.SequenceMediator.mediate(SequenceMediator.java:158)
at org.apache.synapse.mediators.MediatorFaultHandler.onFault(MediatorFaultHandler.java:96)
at org.apache.synapse.FaultHandler.handleFault(FaultHandler.java:53)
at org.apache.synapse.FaultHandler.handleFault(FaultHandler.java:59)
at org.apache.synapse.endpoints.AbstractEndpoint.invokeNextFaultHandler(AbstractEndpoint.java:745)
at org.apache.synapse.endpoints.AbstractEndpoint.onFault(AbstractEndpoint.java:560)
at org.apache.synapse.endpoints.HTTPEndpoint.onFault(HTTPEndpoint.java:67)
at org.apache.synapse.FaultHandler.handleFault(FaultHandler.java:53)
at org.apache.synapse.core.axis2.TimeoutHandler.processCallbacks(TimeoutHandler.java:185)
at org.apache.synapse.core.axis2.TimeoutHandler.run(TimeoutHandler.java:90)
at java.base/java.util.TimerThread.mainLoop(Timer.java:556)
at java.base/java.util.TimerThread.run(Timer.java:506)
now i want to separate logs by it's log level and make conditions with if loglevel=info do
if log level = warn this process
and if log level=error do this process
how should i put it like this?