Has anyone created a successful grok pattern to ingest Github audit logs into ELK? Or does the Github plugin support formatting those logs into ELK?
How are you collecting the Github audit logs? As far as I know github audit logs are in JSON, so you should use the json filter, not grok.
I have the following flow:
Github log stream to bucket -> Collector Script -> Kafka -> Logstash -> Elasticsearch
My parse is the following:
# pipeline: github-audit
# filter
#
filter {
mutate {
add_field => {
"[event][original]" => "%{message}"
}
}
json {
source => "message"
target => "_github"
remove_field => ["message"]
}
}
#
filter {
if [_github][created_at] {
date {
match => ["[_github][created_at]", "UNIX_MS"]
target => "@timestamp"
remove_field => ["[_github][created_at]"]
}
}
if [_github][@timestamp] {
date {
match => ["[_github][@timestamp]", "UNIX_MS"]
target => "@timestamp"
remove_field => ["[_github][@timestamp]"]
}
}
mutate {
rename => {
"[_github][_document_id]" => "[event][id]"
"[_github][action]" => "[event][action]"
"[_github][actor]" => "[user][name]"
"[_github][actor_location][country_code]" => "[client][geo][country_iso_code]"
"[_github][org]" => "[github][org]"
"[_github][user]" => "[user][target][name]"
"[_github][repo]" => "[github][repo]"
"[_github][team]" => "[github][team]"
}
}
if [event][action] {
dissect {
mapping => {
"[event][action]" => "%{[github][category]}.%{[github][action]}"
}
}
}
mutate {
rename => {
"[_github][business]" => "[github][business][name]"
"[_github][business_id]" => "[github][business][id]"
"[_github][user_agent]" => "[user_agent][original]"
"[_github][repository]" => "[github][repository][name]"
"[_github][repository_public]" => "[github][repository][public]"
"[_github][repo_id]" => "[github][repository][id]"
"[_github][visibility]" => "[github][repository][visibility]"
"[_github][previous_visibility]" => "[github][repository][previous_visibility]"
"[_github][public_repo]" => "[github][repository][public]"
"[_github][branch]" => "[github][repository][branch]"
"[_github][owner]" => "[github][repository][owner]"
"[_github][ghsa_id]" => "[github][ghsa_id]"
"[_github][alert_number]" => "[github][alert_number]"
"[_github][transport_protocol_name]" => "[network][protocol]"
"[_github][user_id]" => "[user][target][id]"
"[_github][integration]" => "[github][integration]"
"[_github][actor_id]" => "[user][id]"
"[_github][operation_type]" => "[github][operation_type]"
"[_github][oauth_application]" => "[github][oauth_application]"
"[_github][oauth_application_id]" => "[github][oauth_application_id]"
"[_github][programmatic_access_type]" => "[github][programmatic_access_type]"
"[_github][org_id]" => "[github][org_id]"
"[_github][token_scopes]" => "[token][scopes]"
"[_github][token_id]" => "[token][id]"
"[_github][hashed_token]" => "[token][hash][sha256]"
"[_github][pull_request_title]" => "[github][pull_request][title]"
"[_github][pull_request_id]" => "[github][pull_request][id]"
"[_github][pull_request_url]" => "[github][pull_request][url]"
"[_github][runner_name]" => "[github][runner][name]"
"[_github][runner_id]" => "[github][runner][id]"
"[_github][runner_group_name]" => "[github][runner][group_name]"
"[_github][runner_group_id]" => "[github][runner][group_id]"
"[_github][runner_labels]" => "[github][runner][labels]"
"[_github][runner_owner_type]" => "[github][runner][owner_type]"
"[_github][is_hosted_runner]" => "[github][runner][is_hosted]"
"[_github][source_version]" => "[github][runner][source_version]"
"[_github][target_version]" => "[github][runner][target_version]"
"[_github][actions_cache_id]" => "[github][actions_cache][id]"
"[_github][actions_cache_key]" => "[github][actions_cache][key]"
"[_github][actions_cache_scope]" => "[github][actions_cache][scope]"
"[_github][actions_cache_version]" => "[github][actions_cache][version]"
"[_github][new_repo_permission]" => "[github][new_repo_permission]"
"[_github][old_repo_permission]" => "[github][old_repo_permission]"
"[_github][old_permissions]" => "[github][old_permissions]"
"[_github][environment_name]" => "[github][environment_name]"
"[_github][workflow_id]" => "[github][workflow][id]"
"[_github][workflow_run_id]" => "[github][workflow][run_id]"
"[_github][trigger_id]" => "[github][workflow][trigger_id]"
"[_github][event]" => "[github][workflow][trigger_event]"
"[_github][head_branch]" => "[github][workflow][head_branch]"
"[_github][head_sha]" => "[github][workflow][head_sha]"
"[_github][started_at]" => "[github][workflow][started_at]"
"[_github][cancelled_at]" => "[github][workflow][cancelled_at]"
"[_github][completed_at]" => "[github][workflow][completed_at]"
"[_github][job_name]" => "[github][workflow][job_name]"
"[_github][job_workflow_ref]" => "[github][workflow][job_workflow_ref]"
"[_github][secrets_passed]" => "[github][workflow][secrets_passed]"
"[_github][run_attempt]" => "[github][workflow][run_attempt]"
"[_github][run_number]" => "[github][workflow][run_number]"
"[_github][check_run_id]" => "[github][workflow][check_run_id]"
"[_github][rerun_type]" => "[github][workflow][rerun_type]"
"[_github][conclusion]" => "[github][workflow][conclusion]"
"[_github][calling_workflow_refs]" => "[github][workflow][calling_workflow_refs]"
"[_github][calling_workflow_shas]" => "[github][workflow][calling_workflow_shas]"
"[_github][hook_id]" => "[github][hook][id]"
"[_github][events]" => "[github][hook][events]"
"[_github][events_were]" => "[github][hook][events]"
"[_github][active]" => "[github][hook][active]"
"[_github][active_was]" => "[github][hook][active_was]"
"[_github][config][url]" => "[github][hook][config][url]"
"[_github][config][secret]" => "[github][hook][config][secret]"
"[_github][config][content_type]" => "[github][hook][config][content_type]"
"[_github][config][content-type]" => "[github][hook][config][content_type]"
"[_github][config][insecure_ssl]" => "[github][hook][config][insecure_ssl]"
"[_github][config_was][url]" => "[github][hook][config_was][url]"
"[_github][config_was][secret]" => "[github][hook][config_was][secret]"
"[_github][config_was][content_type]" => "[github][hook][config_was][content_type]"
"[_github][config_was][content-type]" => "[github][hook][config_was][content_type]"
"[_github][config_was][insecure_ssl]" => "[github][hook][config_was][insecure_ssl]"
"[_github][repository_selection]" => "[github][repository_selection]"
"[_github][repositories_added]" => "[github][repositories_added]"
"[_github][repositories_added_names]" => "[github][repositories_added_names]"
"[_github][repositories_removed]" => "[github][repositories_removed]"
"[_github][repositories_removed_names]" => "[github][repositories_removed_names]"
"[_github][inviter]" => "[github][inviter]"
"[_github][invitee]" => "[github][invitee]"
"[_github][invitee_email]" => "[github][invitee_email]"
"[_github][permission]" => "[github][permission]"
"[_github][old_permission]" => "[github][old_permission]"
"[_github][package]" => "[github][package][name]"
"[_github][version]" => "[github][package][version]"
"[_github][ecosystem]" => "[github][package][ecosystem]"
"[_github][is_republished]" => "[github][package][is_republished]"
}
}
# fields related to the protected_branch category
mutate {
rename => {
"[_github][signature_requirement_enforcement_level]" => "[github][signature_requirement_enforcement_level]"
"[_github][allow_deletions_enforcement_level]" => "[github][allow_deletions_enforcement_level]"
"[_github][linear_history_requirement_enforcement_level]" => "[github][linear_history_requirement_enforcement_level]"
"[_github][ignore_approvals_from_contributors]" => "[github][ignore_approvals_from_contributors]"
"[_github][merge_queue_enforcement_level]" => "[github][merge_queue_enforcement_level]"
"[_github][lock_allows_fetch_and_merge]" => "[github][lock_allows_fetch_and_merge]"
"[_github][require_code_owner_review]" => "[github][require_code_owner_review]"
"[_github][required_approving_review_count]" => "[github][required_approving_review_count]"
"[_github][enforcement_level]" => "[github][enforcement_level]"
"[_github][require_last_push_approval]" => "[github][require_last_push_approval]"
"[_github][strict_required_status_checks_policy]" => "[github][strict_required_status_checks_policy]"
"[_github][required_status_checks_enforcement_level]" => "[github][required_status_checks_enforcement_level]"
"[_github][authorized_actor_names]" => "[github][authorized_actor_names]"
"[_github][admin_enforced]" => "[github][admin_enforced]"
"[_github][required_review_thread_resolution_enforcement_level]" => "[github][required_review_thread_resolution_enforcement_level]"
"[_github][required_deployments_enforcement_level]" => "[github][required_deployments_enforcement_level]"
"[_github][create_protected]" => "[github][create_protected]"
"[_github][pull_request_reviews_enforcement_level]" => "[github][pull_request_reviews_enforcement_level]"
"[_github][lock_branch_enforcement_level]" => "[github][lock_branch_enforcement_level]"
"[_github][dismiss_stale_reviews_on_push]" => "[github][dismiss_stale_reviews_on_push]"
"[_github][allow_force_pushes_enforcement_level]" => "[github][allow_force_pushes_enforcement_level]"
"[_github][overridden_codes]" => "[github][protected][overridden_codes]"
"[_github][reasons]" => "[github][protected][reasons]"
"[_github][after]" => "[github][protected][sha_after]"
"[_github][before]" => "[github][protected][sha_before]"
}
}
}
#
filter {
mutate {
split => {
"[token][scopes]" => ","
}
}
mutate {
remove_field => [ "[_github][actor_location]", "[_github][transport_protocol]"]
}
ruby {
code => "
event.remove('_github') if event.get('_github').nil? || event.get('_github').empty?
"
}
}
#
filter {
fingerprint {
source => "[event][id]"
method => "SHA1"
target => "[@metadata][fingerprint]"
}
if ![@metadata][fingerprint] {
drop {}
}
}
#
filter {
if [github][category] == "workflows" {
mutate {
rename => {
"[_github][name]" => "[github][workflow][name]"
}
}
}
if [github][category] == "hook" {
mutate {
rename => {
"[_github][name]" => "[github][hook][name]"
}
}
}
if [github][category] in [ "integration_installation", "integration" ] {
mutate {
rename => {
"[_github][name]" => "[github][integration_name]"
}
}
}
if [github][category] == "protected_branch" {
mutate {
rename => {
"[_github][name]" => "[github][repository][branch]"
}
}
}
}
I created this parse based on the github audit elastic integration, but the integration only parses a couple of fields, the majority of them would probably be dropped.
1 Like
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.