Parsing confluence audit json into Elastic with logstash

Elastic Stack Logstash
1

Hello,

I'm testing scraping our company confluence (self hosted) audit api so permission changes etc. are in ELK (also self hosted).

I've got an http_poller pipeline that is successfully polling the api, however the event shows up in elastic as a giant blob with the fields in an unreadable order.

If I curl the api and pipe to a file, the format looks like this (truncated for brevity)-

{
  "results": [
    {
      "author": {
        "type": "user",
        "displayName": "admin",
        "username": "admin",
        "userKey": ""
      },
      "remoteAddress": "192.168.1.171",
      "creationDate": 1733800689935,
      "summary": "Audit Log search performed",
      "description": "",
      "category": "Auditing",
      "sysAdmin": false,
      "affectedObject": {
        "name": "",
        "objectType": ""
      },
      "changedValues": [],
      "associatedObjects": []
    },
    {
      "author": {
        "type": "user",
        "displayName": "System",
        "userKey": "-1"
      },
      "creationDate": 1700046445029,
      "summary": "User added to group",
      "description": "",
      "category": "Users and groups",
      "sysAdmin": false,
      "affectedObject": {
        "name": "My_Group_Name",
        "objectType": "Group"
      },
      "changedValues": [],
      "associatedObjects": [
        {
          "name": "john.smith",
          "objectType": "User"
        }
      ]
    }
	],
	"start": 0,
  "limit": 1000,
  "size": 1000,
  "_links": {
    "self": "https://confluence.domain.net:8443/rest/api/audit",
    "base": "https://confluence.domain.net:8443",
    "context": ""
  }
}

The pipeline config looks like this

input {
  http_poller {
    urls => {
      confluence => {
        method => get
        url => "https://confluence.domain.net:8443/rest/api/audit"
        headers => {
          Authorization => "Bearer <access token here>/p"
          Accept => "application/json"
        }
     }
    }
    truststore => "/etc/logstash/config/certs/trusted_certs.jks"
    truststore_password => "redacted"
    request_timeout => 60
    # Supports "cron", "every", "at" and "in" schedules by rufus scheduler
    schedule => { cron => "*/5 * * * *"}
    codec => "json"
    # A hash of request metadata info (timing, response headers, etc.) will be sent here
    # metadata_target => "http_poller_metadata"
  }
}

output {
  elasticsearch {
    hosts => ["https://127.0.0.1:9200"]
    index => "confluence-%{+YYYY.MM.dd}"
    user => "logstash-writer"
    password => "redacted"
    ssl => "true"
    ssl_certificate_verification => "false"
    cacert => "/etc/logstash/config/certs/http_ca.crt"
    codec => "json"
  }
}

When I search the index in kibana, the output looks like this

{
  "_links.base": [
    "https://confluence.domain.net:8443"
  ],
  "_links.base.keyword": [
    "https://confluence.domain.net:8443"
  ],
  "_links.context": [
    ""
  ],
  "_links.context.keyword": [
    ""
  ],
  "_links.self": [
    "https://confluence.domain.net:8443/rest/api/audit"
  ],
  "_links.self.keyword": [
    "https://confluence.domain.net:8443/rest/api/audit"
  ],
  "@timestamp": [
    "2024-12-11T01:35:00.785Z"
  ],
  "@version": [
    "1"
  ],
  "@version.keyword": [
    "1"
  ],
  "event.original": [
    "{\"results\":[{\"author\":{\"type\":\"user\",\"displayName\":\"admin\",\"username\":\"admin\",\"userKey\":\""\"},\"remoteAddress\":\"192.168.1.172\",\"creationDate\":1733880760520,\"summary\":\"Audit Log search performed\",\"description\":\"\",\"category\":\"Auditing\",\"sysAdmin\":false,\"affectedObject\":{\"name\":\"\",\"objectType\":\"\"},\"changedValues\":[],\"associatedObjects\":[]},{\"author\":{\"type\":\"user\",\"displayName\":\"admin\",\"username\":\"admin\",\"userKey\":\"\"},\"remoteAddress\":\"192.168.1.172\",\"creationDate\":1733880460633,\"summary\":\"Audit Log search performed\",\"description\":\"\",\"category\":\"Auditing\",\"sysAdmin\":false,\"affectedObject\":{\"name\":\"\",\"objectType\":\"\"},\"changedValues\":[],\"associatedObjects\":[]}],\"start\":0,\"limit\":2,\"size\":2,\"_links\":{\"self\":\"https://confluence.domain.net:8443/rest/api/audit\",\"base\":\"https://confluence.domain.net:8443\",\"context\":\"\"}}"
  ],
  "event.original.keyword": [
    "{\"results\":[{\"author\":{\"type\":\"user\",\"displayName\":\"admin\",\"username\":\"admin\",\"userKey\":\"\"},\"remoteAddress\":\"192.168.1.172\",\"creationDate\":1733880760520,\"summary\":\"Audit Log search performed\",\"description\":\"\",\"category\":\"Auditing\",\"sysAdmin\":false,\"affectedObject\":{\"name\":\"\",\"objectType\":\"\"},\"changedValues\":[],\"associatedObjects\":[]},{\"author\":{\"type\":\"user\",\"displayName\":\"admin\",\"username\":\"admin\",\"userKey\":\"\"},\"remoteAddress\":\"192.168.1.172\",\"creationDate\":1733880460633,\"summary\":\"Audit Log search performed\",\"description\":\"\",\"category\":\"Auditing\",\"sysAdmin\":false,\"affectedObject\":{\"name\":\"\",\"objectType\":\"\"},\"changedValues\":[],\"associatedObjects\":[]}],\"start\":0,\"limit\":2,\"size\":2,\"_links\":{\"self\":\"https://confluence.domain.net:8443/rest/api/audit\",\"base\":\"https://confluence.domain.net:8443\",\"context\":\"\"}}"
  ],
  "limit": [
    2
  ],
  "results.affectedObject.name": [
    "",
    ""
  ],
  "results.affectedObject.name.keyword": [
    "",
    ""
  ],
  "results.affectedObject.objectType": [
    "",
    ""
  ],
  "results.affectedObject.objectType.keyword": [
    "",
    ""
  ],
  "results.author.displayName": [
    "admin",
    "admin"
  ],
  "results.author.displayName.keyword": [
    "admin",
    "admin"
  ],
  "results.author.type": [
    "user",
    "user"
  ],
  "results.author.type.keyword": [
    "user",
    "user"
  ],
  "results.author.userKey": [
    "",
    ""
  ],
  "results.author.userKey.keyword": [
    "",
    ""
  ],
  "results.author.username": [
    "admin",
    "admin"
  ],
  "results.author.username.keyword": [
    "admin",
    "admin"
  ],
  "results.category": [
    "Auditing",
    "Auditing"
  ],
  "results.category.keyword": [
    "Auditing",
    "Auditing"
  ],
  "results.creationDate": [
    1733880760520,
    1733880460633
  ],
  "results.description": [
    "",
    ""
  ],
  "results.description.keyword": [
    "",
    ""
  ],
  "results.remoteAddress": [
    "192.168.1.172",
    "192.168.1.172"
  ],
  "results.remoteAddress.keyword": [
    "192.168.1.172",
    "192.168.1.172"
  ],
  "results.summary": [
    "Audit Log search performed",
    "Audit Log search performed"
  ],
  "results.summary.keyword": [
    "Audit Log search performed",
    "Audit Log search performed"
  ],
  "results.sysAdmin": [
    false,
    false
  ],
  "size": [
    2
  ],
  "start": [
    0
  ],
  "_id": "Impas5MBRqo3Lxj_7boe",
  "_ignored": [
    "event.original.keyword"
  ],
  "_index": "confluence-2024.12.11",
  "_score": null
}

Is there a way to get each result from the json as an individual event in elastic? Ie. is there a way to get

{
      "author": {
        "type": "user",
        "displayName": "System",
        "userKey": "-1"
      },
      "creationDate": 1700046445029,
      "summary": "User added to group",
      "description": "",
      "category": "Users and groups",
      "sysAdmin": false,
      "affectedObject": {
        "name": "My_Group_Name",
        "objectType": "Group"
      },
      "changedValues": [],
      "associatedObjects": [
        {
          "name": "john.smith",
          "objectType": "User"
        }
      ]
    }

as a single event?

I tried adding a filter, but it didn't seem to make any sort of impact on how the data looked.

    filter {
      json {
        source => "results"
      }
    }
2

So before I dive into your technical details. Are you aware of the Atlassian Confluence integration for Fleet which does all the work for you?

Post that for logstash I am pretty sure you are looking for the split plugin.

filter {
 split {
   field => "results"
 }
}
2 Likes
3

Thank you this is exactly what I was looking for!