**I'm trying to add new hardcoded field in logstash filter:**
My logstash config:
input {
elasticsearch {
hosts => "http://localhost:9200"
index => "ccee_mathematics_academic_indicator_2018"
}
}
filter {
mutate {
remove_field => ["@version","@DATE","column33","column34","column35","column36","path","host","type", "message", "@timestamp"]
add_field => { "my_new_field" => "CDSCode" }
}
}
output {
google_bigquery {
project_id => "xxxxxxx"
dataset => "yyyyyy"
table_prefix => "ElaMathIndicator"
table_separator => ""
date_pattern => ""
flush_interval_secs => 30
json_key_file => "D:/State_Dashboard/key.json"
json_schema => {
fields => [
{ name => "CDSCode" type => "STRING" mode => "NULLABLE"},
{ name => "RecordType" type => "STRING" mode => "NULLABLE"},
{ name => "SchoolName" type => "STRING" mode => "NULLABLE"},
{ name => "DistrictName" type => "STRING" mode => "NULLABLE"},
{ name => "CountyName" type => "STRING" mode => "NULLABLE"},
{ name => "CharterFlag" type => "STRING" mode => "NULLABLE"},
{ name => "CountyOfficeofEducationFlag" type => "STRING" mode => "NULLABLE"},
{ name => "DashboardAlternativeSchoolStatusFlag" type => "STRING" mode => "NULLABLE"},
{ name => "StudentGroup" type => "STRING" mode => "NULLABLE"},
{ name => "currentDenominator" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentDenominatorSWD" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentPercentage" type => "STRING" mode => "NULLABLE"},
{ name => "PreviousdDenominator" type => "STRING" mode => "NULLABLE"},
{ name => "PreviousdDenominatorSWD" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "PreviousPercentage" type => "STRING" mode => "NULLABLE"},
{ name => "Change" type => "STRING" mode => "NULLABLE"},
{ name => "StatusLevel" type => "STRING" mode => "NULLABLE"},
{ name => "ChangeLevel" type => "STRING" mode => "NULLABLE"},
{ name => "Color" type => "STRING" mode => "NULLABLE"},
{ name => "Box" type => "STRING" mode => "NULLABLE"},
{ name => "HScutpoints" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentAdjustment" type => "STRING" mode => "NULLABLE"},
{ name => "priorAdjustment" type => "STRING" mode => "NULLABLE"},
{ name => "PairShareMethod" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentLevelDenominator" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentLevel1" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentPercentageLevel1" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentLevel2" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentPercentageLevel2" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentLevel3" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "CurrentPercentageLevel3" value => "null" type => "STRING" mode => "NULLABLE"},
{ name => "NoTestFlag" type => "STRING" mode => "NULLABLE"},
{ name => "ReportingYear" type => "STRING" mode => "NULLABLE"},
{ name => "my_new_field" type => "STRING" mode => "NULLABLE"}
]
}
error_directory => "D:/ELK_7.9.3"
}
}
error:
[2020-11-02T17:07:21,549][WARN ][logstash.outputs.googlebigquery][main][892ec9d1a68e0181790c9f6072265a01b6695ad5a542addb2dc653a9170567c6] Error while inserting {:key=>127, :location=>"my_new_field", :message=>"no such field.", :reason=>"invalid"}
could anyone help me asap?