Logstash JDBC_INPUT does not push data upon it starts when there's scheduler in its config

Elastic Stack Logstash
1

I have following jdbc_input configured and expect it runs upon logstash starts AND as scheduled as well. However, it did not run upon logstash starts.

I found following old post. It shall run. I have losgtash 8. 13.2.

jdbc {
    jdbc_driver_library => "xxxxx"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${JDBC_URL}"
    jdbc_user => "${DB_USER}"
    jdbc_password => "${DB_PASSWORD}"
    schedule => "0 4 * * 6"
    statement_filepath => "./config/conf/device_query.sql"
    type => "device"
    jdbc_paging_enabled => true
    jdbc_page_size => "10000"
    jdbc_fetch_size => "10000"
    record_last_run => "false"
  }
2

That's not how cron works, so it's not how a Rufus cronline works, so it's not how a jdbc input works. If you want both a cron schedule and an invocation at startup then use two inputs.

The behaviour of the jdbc filters is different. They have to be ready to process events once they are initialized, so the have to do the load at startup. Doing a reload based on a schedule is a bonus feature.

3

Thank you, Badger. My bad, I mixed JDBC_STATIC filter with JDBC_INPUT. Now I have following config (two w/ different schedules and one w/o schedule).

 jdbc {
    jdbc_driver_library => "${DB_JARS}"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${JDBC_URL}"
    jdbc_user => "${DB_USER}"
    jdbc_password => "${DB_PASSWORD}"
    schedule => "${CRON_SCHEDULE}"
    statement_filepath => "./config/conf/sql_incr/device_query.sql"
    type => "device"
    use_column_value => "true"
    tracking_column => "audit_timestamp"
    tracking_column_type => "numeric"
    jdbc_paging_enabled => true
    jdbc_page_size => "10000"
    jdbc_fetch_size => "10000"
    last_run_metadata_path => "${LOGSTASH_TMP_DIR}/.logstash_jdbc_last_run_ts_device"
  }

  # Full push: Run once on start-up and as scheduled
  jdbc {
    jdbc_driver_library => "${DB_JARS}"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${JDBC_URL}"
    jdbc_user => "${DB_USER}"
    jdbc_password => "${DB_PASSWORD}"
    statement_filepath => "./config/conf/${SQL_FULL_PATH}/device_query.sql"
    type => "device"
    jdbc_paging_enabled => true
    jdbc_page_size => "10000"
    jdbc_fetch_size => "10000"
    record_last_run => "false"
  }
  jdbc {
    jdbc_driver_library => "${DB_JARS}"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${JDBC_URL}"
    jdbc_user => "${DB_USER}"
    jdbc_password => "${DB_PASSWORD}"
    schedule => "${FULL_SYNC_CRON_SCHEDULE}"
    statement_filepath => "./config/conf/${SQL_FULL_PATH}/device_query.sql"
    type => "device"
    jdbc_paging_enabled => true
    jdbc_page_size => "10000"
    jdbc_fetch_size => "10000"
    record_last_run => "false"
  }