PingFederate Server Logs Help

Elastic Stack Logstash
1

I'm trying to get a grok pattern working for PingFederate server logs, specifically the instances that have multi-line XML. The logs are coming in via FileBeats which is configured correctly for multiline files. Below is a (redacted) sample:

2019-06-03 16:03:20,463 tid:123abcdef123abcdef-a1c-1ABc DEBUG [org.sourceid.saml20.bindings.LoggingInterceptor] Received InMessageContext:
InMessageContext
XML: <samlp:Response Version="2.0" ID="123abcdef123abcdef.a1c1ABc" IssueInstant="2019-06-03T20:03:20.179Z" InResponseTo="123abcdef123abcdef.a1c1DEf" Destination="https://sso-test.company.com/sp/eyJ2c2lkIjoidXJuOmFjZTpzcDpkdWNrY3JlZWs6dWF0In0/ACS.saml2" xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol">
  <saml:Issuer xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion">urn:test:test:testsite:uat</saml:Issuer>
  <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
    <ds:SignedInfo>
      <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/>
      <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/>
      <ds:Reference URI="#123abcdef123abcdef.a1c1ABc">
        <ds:Transforms>
          <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/>
          <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/>
        </ds:Transforms>
        <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/>
        <ds:DigestValue>fFFrFFaFffS2O9hyBmfg74wFlh05rcLFffff07LWD04=</ds:DigestValue>
      </ds:Reference>
    </ds:SignedInfo>
    <ds:SignatureValue>FFFF+/FFFFwS+yhZGQ7vvZfPZqDv1GzdrZzGeOdj+fffF1xLItB4hea0LVerXfH1RWjoplk5G0VvG3PHfMQb8efEFOxbDrc/CPMO3LCAgS6ToKtO4jGHgs8j98ev8VmJoqEIY1iZQu/UrGwhYMnFmXOeiYv3zaAE9qzz8CdKFgj9F4mWF32yDGgo0m2B7YeonpKLDxWGx8mWm/wsAHO9eZfkA/FuGz087m3LR5S6xrtjQpvmqUj8IXdS3GsYmbBGQgTul3bDt9OKdjbFFFFFF+J0iOxciIMixerK8NXqHzrjOTUZfVLPb4RQluwX4lUiJc++mMjG8Q2rdyY/YQ==</ds:SignatureValue>
  </ds:Signature>
  <samlp:Status>
    <samlp:StatusCode Value="urn:oasis:names:tc:SAML:2.0:status:Success"/>
  </samlp:Status>
  <saml:Assertion ID="FFffffFFFFFFFFFffff" IssueInstant="2019-06-03T20:03:20.273Z" Version="2.0" xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion">
    <saml:Issuer>urn:test:test:testsite:uat</saml:Issuer>
    <saml:Subject>
      <saml:NameID Format="urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified">TESTUSER01</saml:NameID>
      <saml:SubjectConfirmation Method="urn:oasis:names:tc:SAML:2.0:cm:bearer">
        <saml:SubjectConfirmationData Recipient="https://sso-test.company.com/sp/fffeFFFFFefefeffffFFFFFF0123FFfffff/ACS.saml2" NotOnOrAfter="2019-06-03T20:08:20.273Z" InResponseTo="ffFFFFffFFF_f.KHCPfswQ"/>
      </saml:SubjectConfirmation>
    </saml:Subject>
    <saml:Conditions NotBefore="2019-06-03T19:58:20.273Z" NotOnOrAfter="2019-06-03T20:08:20.273Z">
      <saml:AudienceRestriction>
        <saml:Audience>urn:ace:sp:duckcreek:uat</saml:Audience>
      </saml:AudienceRestriction>
    </saml:Conditions>
    <saml:AuthnStatement SessionIndex="FFff9ffFl1wobU4p5M_oiWlTTwN" AuthnInstant="2019-06-03T20:03:20.257Z">
      <saml:AuthnContext>
        
      </saml:AuthnContext>
    </saml:AuthnStatement>
    <saml:AttributeStatement>
      <saml:Attribute Name="firstname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
        <saml:AttributeValue xsi:type="xs:string" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">TESTUSER01</saml:AttributeValue>
      </saml:Attribute>
      <saml:Attribute Name="domain" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
        <saml:AttributeValue xsi:type="xs:string" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">USER01</saml:AttributeValue>
      </saml:Attribute>
      <saml:Attribute Name="objectGUID" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
        <saml:AttributeValue xsi:type="xs:string" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">fffFFFfffFFfFffffff==</saml:AttributeValue>
      </saml:Attribute>
      <saml:Attribute Name="lanid" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
        <saml:AttributeValue xsi:type="xs:string" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">TESTUSER01</saml:AttributeValue>
      </saml:Attribute>
      <saml:Attribute Name="lastname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:basic">
        <saml:AttributeValue xsi:type="xs:string" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">TESTUSER01</saml:AttributeValue>
      </saml:Attribute>
    </saml:AttributeStatement>
  </saml:Assertion>
</samlp:Response>
entityId: urn:test:test:testsite:uat (IDP)
virtualServerId: urn:test:test:testsite:uat
Binding: urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST
relayState: fffFFF123ffFfffFFfffffFFff
SignatureStatus: VALID
Binding says to sign: true

Here is the relevant section of my logstash config:

grok {
          match => { "message" => "%{TIMESTAMP_ISO8601:logdate}\s(?<tid>[^\s]*)?\s%{WORD:loglevel}\s\[(?<sourceid>[^\s]*)\]\s(?<messageType>[a-zA-Z\s\.]*)?\s(In|Out)MessageContext:\r\n(In|Out)MessageContext\r\nXML:\s(?<xmldata>(.|\r|\n)*)\r\nentityId: (?<entityId>[^\r\n]*)\r\n(virtualServerId: (?<virtualServerId>[^\r\n]*)\r\n)?(Binding: (?<binding>[^\r\n]*)\r\n)?(relayState: (?<relayState>[^\r\n]*)\r\n)?(SignatureStatus: %{WORD:signatureStatus}\r\n)?(Binding says to sign: %{WORD:binding_says_to_the_sign})?" }
    }
    date {
      match => [ "logdate", "yyyy-MM-dd HH:mm:ss,SSS" ]
    }
    xml {
      source => "xmldata"
      target => "body"
      #remove_field => ["xmldata"]
    }
  }

Here's the output from the grok debugger:

{
  "sourceid": "org.sourceid.saml20.bindings.LoggingInterceptor",
  "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST",
  "entityId": "urn:test:test:testsite:uat (IDP)",
  "binding_says_to_the_sign": "true",
  "xmldata": "<samlp:Response Version=\"2.0\" >[truncated due to char limit</samlp:Response>",
  "tid": "tid:123abcdef123abcdef-a1c-1ABc",
  "virtualServerId": "urn:test:test:testsite:uat",
  "relayState": "fffFFF123ffFfffFFfffffFFff",
  "signatureStatus": "VALID",
  "messageType": "Received",
  "logdate": "2019-06-03 16:03:20,463",
  "loglevel": "DEBUG"
}

I've tested this in the grok debugger and it seems to work fine, but I always get a _grokparsefailure. Any clue as to what I'm doing wrong here?

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.