Convert strings with different data units (MB,GB,TB) to byte

Elastic Stack Elasticsearch
6

You could manage it in Logstash. Here's an example from something I did; your approach would have to vary slightly (might actually be easier, since this example is for strings with units appended without any whitespace.

# split the "store" into number, units prefix, and units base
    grok {
        # would like to put store_units_prefix and store_units_base in @metadata, too
        match => { "store" => "^%{BASE10NUM:[@metadata][store_number]:float}(?<store_units_prefix>[kKmMgGtT])(?<store_units_base>[b])$" }
    }
    mutate {
        add_field => {
            "[@metadata][store_units_prefix]" => "%{store_units_prefix}"
            "[@metadata][store_units_base]" => "%{store_units_base}"
        }
        remove_field => [ "store_units_prefix", "store_units_base" ]
    }
    if [@metadata][store_units_prefix] == "k" or [@metadata][store_units_prefix] == "K" {
        mutate { add_field => { "[@metadata][store_multiplier]" => 1024 } }
    } else if [@metadata][store_units_prefix] == "m" or [@metadata][store_units_prefix] == "M" {
        mutate { add_field => { "[@metadata][store_multiplier]" => 1048576 } }
    } else if [@metadata][store_units_prefix] == "g" or [@metadata][store_units_prefix] == "G" {
        mutate { add_field => { "[@metadata][store_multiplier]" => 1073741824 } }
    } else if [@metadata][store_units_prefix] == "t" or [@metadata][store_units_prefix] == "T" {
        mutate { add_field => { "[@metadata][store_multiplier]" => 1099511627776 } }
    }
# I don't know how to specify type in mutate.add_field, so I convert it
    mutate {
        convert => { "[@metadata][store_multiplier]" => "integer" }
    }
# create a new field with the size in bytes
    ruby {
        code => "event['store_size'] = event['@metadata']['store_number']*event['@metadata']['store_multiplier']"
    }
4 Likes