Addition of response time into Apache access fileset

[jberanek (John Beranek)]

So, I'm not sure how specific this is to our environment, but our Apache logs have response time added onto the end of one of filebeat's 'apache' module's standard formats, like:

2s 2123456 micro-seconds

So, I modified the filebeat-X-apache-access-default pipeline to be able to pull out this information, applying it to the ECS field event.duration

I'm not totally sure on the syntax in the filebeat pipeline.yml, but here's my attempt:

    $ diff -u /usr/share/filebeat/module/apache/access/ingest/pipeline.yml  pipeline.yml
    --- /usr/share/filebeat/module/apache/access/ingest/pipeline.yml        2020-12-04 23:27:07.000000000 +0000
    +++ pipeline.yml        2021-01-08 13:50:39.507533000 +0000
    @@ -14,6 +14,11 @@
         - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
           "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
           %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
    +      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?
    +      %{NUMBER:event.duration_seconds}s %{NUMBER:event.duration_microseconds} micro-seconds'
    +    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
    +      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
    +      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
           "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
         - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
           "-" %{NUMBER:http.response.status_code:long} -'
    @@ -96,6 +101,17 @@
             ctx.tls.version = parts[1] + ".0";
           }
           ctx.tls.version_protocol = parts[0];
    +- script:
    +    lang: painless
    +    if: ctx?.event?.duration_microseconds != null
    +    source: >-
    +      long ms = Integer.parseInt(ctx.event.duration_microseconds);
    +      ctx.event.duration = ms*1000;
    +- remove:
    +    field:
    +     - event.duration_seconds
    +     - event.duration_microseconds
    +    ignore_failure: true

     on_failure:
     - set:

Is this of value?

[jberanek (John Beranek)]

Full pipeline.yml, based on filebeat 7.10.1:

description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."

processors:
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- grok:
    field: message
    patterns:
    - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?
      %{NUMBER:event.duration_seconds}s %{NUMBER:event.duration_microseconds} micro-seconds'
    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "-" %{NUMBER:http.response.status_code:long} -'
    - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original}
      HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
    ignore_missing: true
- remove:
    field: message
- set:
    field: event.kind
    value: event
- set:
    field: event.category
    value: web
- set:
    field: event.outcome
    value: success
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
- set:
    field: event.outcome
    value: failure
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
- grok:
    field: source.address
    ignore_missing: true
    patterns:
    - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
- rename:
    field: '@timestamp'
    target_field: event.created
- date:
    field: apache.access.time
    target_field: '@timestamp'
    formats:
    - dd/MMM/yyyy:H:m:s Z
    ignore_failure: true
- remove:
    field: apache.access.time
    ignore_failure: true
- user_agent:
    field: user_agent.original
    ignore_failure: true
- geoip:
    field: source.ip
    target_field: source.geo
    ignore_missing: true
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- rename:
    field: source.as.asn
    target_field: source.as.number
    ignore_missing: true
- rename:
    field: source.as.organization_name
    target_field: source.as.organization.name
    ignore_missing: true
- set:
    field: tls.cipher
    value: '{{apache.access.ssl.cipher}}'
    ignore_empty_value: true

- script:
    lang: painless
    if: ctx?.apache?.access?.ssl?.protocol != null
    source: >-
      def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
      if (parts.length != 2) {
        return;
      }
      if (parts[1].contains(".")) {
        ctx.tls.version = parts[1];
      } else {
        ctx.tls.version = parts[1] + ".0";
      }
      ctx.tls.version_protocol = parts[0];
- script:
    lang: painless
    if: ctx?.event?.duration_microseconds != null
    source: >-
      long ms = Integer.parseInt(ctx.event.duration_microseconds);
      ctx.event.duration = ms*1000;
- remove:
    field:
     - event.duration_seconds
     - event.duration_microseconds
    ignore_failure: true

on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'