Logstash custom pattern for grok filter

Elastic Stack Logstash
1

We have a log file in which we are reading specific lines and storing it against a field for that we are using below mentioned config but it's look lengthy, repetitive and not professional but serves the purpose. I hope there is a better way to do that instead adding each grok for each line, Please suggest us
Note: I didn't post the full logstash config but posted the repeated part

Log File EX:
[2019-02-12 09:16:51Z INFO Program] Version: 2.122.1
[2019-02-12 09:16:51Z INFO Program] Commit: fc6746af2429c6ffa73c0793d732e6853f9fb375
[2019-02-12 09:16:51Z INFO Program] Culture: de-CH
[2019-02-12 09:16:51Z INFO Program] UI Culture: en-US
[2019-02-12 09:16:51Z INFO VstsAgentWebProxy] No proxy setting found.
[2019-02-12 09:16:51Z INFO Worker] Waiting to receive the job message from the channel.
[2019-02-12 09:16:51Z INFO Worker] Message received.
[2019-02-12 09:16:52Z INFO Worker] Job message:
{
"tasks": [
{
"variables": {
"system": "build",
"system.collectionId": "971eadf7-7e5c-4fe9-a411-db960ff4ce26",
"system.teamProject": "Our project",
"system.teamProjectId": "5ffc7a09-3b4d-4f99-b741-e06b0284b0ae",
"system.definitionId": "19883",
"build.definitionName": "Our build def",
"build.definitionVersion": "18",
"build.queuedBy": "Our user",
"build.queuedById": "e48dc211-5757-454a-9862-687bf788c63e",
"build.requestedFor": "Our user",
"build.requestedForId": "e48dc211-5757-454a-9862-687bf788c63e",
"build.requestedForEmail": "Our user.com",
"build.sourceVersion": "838834",
"build.sourceBranch": "master",
"build.sourceBranchName": "rb01",
"build.sourceTfvcShelveset": "32fwqf1f",
"system.debug": "false",
"AssemblyFileVersion": "01.04.00",
"BuildName": "SQA",
"HypervName": "Our HypervName",
"HypervVersion": "V32",
"BuildDropForMCD": "Our build drop",
"SONAR_SCANNER_OPTS": "-Xmx4096m",
"build.reason": "Manual",
"build.buildId": "1467340",
"build.buildUri": "vstfs:///ourUri",
"build.buildNumber": "Our Build number",
"build.containerId": "1218498",
"system.isScheduled": "False",
"system.hosttype": "build",
"system.culture": "en-US",
"system.teamFoundationCollectionUri": "OurProjectURI",
"system.taskDefinitionsUri": "OurProjectBuildDefURI",
"AZURE_HTTP_USER_AGENT": "TFS_971eadf7-7e5c-4fe9-a411-db960ff4ce26_build_19883_1467340",
"MSDEPLOY_HTTP_USER_AGENT": "TFS_971eadf7-7e5c-4fe9-a411-db960ff4ce26_build_19883_1467340",
"system.planId": "e454a646-71bf-4071-b436-aa36f912032b",
"system.jobId": "0ce068ad-6645-4cf9-b309-661eba2beb71",
"system.timelineId": "8fe7063d-4336-4b5b-876d-5e3d7e414678",
"build.repository.uri": "OurProjectURI",
"build.sourceVersionAuthor": "Our Author",
"build.sourceVersionMessage": "No dependency"
}
}

Log stash config:
input {
beats {
client_inactivity_timeout => 1200
port => 5002
}
}
filter
{
grok {
add_tag => [ "start" ]
match => { "message" => "%{TIMESTAMP_ISO8601:build_StartTime}" }
}
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..definitionName": "(?<build_DefinitionName>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..requestedFor": "(?<build_RequesterName>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..buildNumber": "(?<build_BuildNumber>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..teamProject": "(?<build_TeamProject>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => "(?<build_ErrorMessage> ERR .)" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..sourceVersion": "(?<build_SourceVersion>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..sourceBranchName": "(?<build_SourceBranchName>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..buildId": "(?<build_BuildId>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..reason": "(?<build_Reason>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..teamFoundationCollectionUri": "(?<build_TeamFoundationCollectionUri>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..repository.uri": "(?<build_RepositoryUri>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..sourceVersionAuthor": "(?<build_SourceVersionAuthor>.?)"" }
}
grok {
add_tag => [ "start" ]
match => { "message" => ""..sourceVersionMessage": "(?<build_SourceVersionMessage>.*?)"" }
}
}

2

You can use an array of patterns to match. They will be a lot more efficient if you anchor them to start of line using ^

    grok {
        add_tag => [ "start" ]
        match => {
            "message" => [
                "^\[%{TIMESTAMP_ISO8601:build_StartTime}",
                '^"%{WORD}\.definitionName": "(?<build_DefinitionName>.+)"',
                '^"%{WORD}\.requestedFor": "(?<build_RequesterName>.+)"',
                '^"%{WORD}\.buildNumber": "(?<build_BuildNumber>.+)"',
                '^"%{WORD}\.teamProject": "(?<build_TeamProject>.+)"',
                '^"%{WORD}\.sourceVersion": "(?<build_SourceVersion>.+)"',
                '^"%{WORD}\.sourceBranchName": "(?<build_SourceBranchName>.+)"',
                '^"%{WORD}\.buildId": "(?<build_BuildId>.+)"',
                '^"%{WORD}\.reason": "(?<build_Reason>.+)"',
                '^"%{WORD}\.teamFoundationCollectionUri": "(?<build_TeamFoundationCollectionUri>.+)"',
                '^"%{WORD}\.repository.uri": "(?<build_RepositoryUri>.+)"',
                '^"%{WORD}\.sourceVersionAuthor": "(?<build_SourceVersionAuthor>.+)"',
                '^"%{WORD}\.sourceVersionMessage": "(?<build_SourceVersionMessage>.+)"'
            ]
        }
        tag_on_failure => []
    }
3

Thanks for the suggestion Let us try this out and get back to you. :grinning:

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.