It appears that the second database call is getting data from the first one.
32-db.conf
input {
jdbc {
#jdbc_driver_library => "/etc/logstash/mysql-connector-java-8.0.18.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://mikesdevhub.com:3306/fish"
jdbc_user => "fish"
jdbc_password => "password"
statement => "SELECT * FROM readings"
schedule => "*/5 * * * *"
clean_run => true
}
}
output {
elasticsearch {
index => "fishstats"
action => "index"
document_id => "%{readings_id}"
hosts => "http://localhost:9200"
}
}
Here is 33-ARC.conf
input {
jdbc {
#jdbc_driver_library => "/etc/logstash/mysql-connector-java-8.0.18.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://mikesdevhub.com:3306/arc1"
jdbc_user => "arc1"
jdbc_password => "password"
statement => "SELECT * FROM applications"
schedule => "*/5 * * * *"
clean_run => true
}
}
output {
elasticsearch {
index => "arcapps"
action => "index"
document_id => "%{app_id}"
hosts => "http://localhost:9200"
}
}
Both of these are pulling data and when I go into the index management it is showing 107 docs for the arcapps index. For the fish index there are 9 docs. When I go to the discover tab I am seeing 1 doc for the arcapps index and 8 docs for the fishstats index.
I am confused as to why the numbers are off but they are only off by 1.
Here is the json for the 1 record under the arcapp index
{
"_index": "arcapps",
"_type": "doc",
"_id": "%{app_id}",
"_version": 104,
"_score": null,
"_source": {
"readings_id": 11,
"date": "2019-11-11T05:00:00.000Z",
"ph": null,
"salinity": 1.0240000486373901,
"reader": null,
"temp": 78.4000015258789,
"nitrates": null,
"ammonia": null,
"@version": "1",
"nitrites": null,
"@timestamp": "2019-11-20T19:15:00.656Z"
},
"fields": {
"date": [
"2019-11-11T05:00:00.000Z"
],
"@timestamp": [
"2019-11-20T19:15:00.656Z"
]
},
"sort": [
1574277300656
]
}
You can see that the data is from two different pulls.
Here is the structure for the arc database
CREATE TABLE `applications` (
`app_id` int(11) NOT NULL,
`mgt_id` int(11) NOT NULL,
`resub` tinyint(1) NOT NULL,
`month_sub` int(11) NOT NULL,
`Date_of_Rec` date NOT NULL,
`Date_of_FP` date NOT NULL,
`Date_Resp_Req` date NOT NULL,
`Resale` tinyint(1) NOT NULL,
`Violation` tinyint(1) NOT NULL,
`VA/OS` enum('VA','OS') NOT NULL,
`Address` varchar(80) NOT NULL,
`Section` varchar(5) NOT NULL,
`Final_Pics` tinyint(1) NOT NULL,
`Request` varchar(50) NOT NULL,
`Home_Type` enum('TH','SFH') NOT NULL,
`Steve_Walters` varchar(30) NOT NULL,
`Antonio_Alamio` varchar(30) NOT NULL,
`Gerald_Bright` varchar(30) NOT NULL,
`Michael_Flack` varchar(30) NOT NULL,
`Brian_Kwesiga` varchar(30) NOT NULL,
`Michael_Brown` varchar(30) NOT NULL,
`Jessica_Arseneault` varchar(30) NOT NULL,
`Jayla_Walters` varchar(30) NOT NULL,
`Tim_Swigert` varchar(30) NOT NULL,
`David_Gurule` varchar(30) NOT NULL,
`Final_Decision` enum('Approved','Denied','In Progress','Stipulations','Incomplete') NOT NULL,
`Group_Decision_Date` date NOT NULL,
`Days_In_Progress` float NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Here is the structure for the fish readings
CREATE TABLE `readings` (
`readings_id` int(11) NOT NULL,
`date` datetime NOT NULL,
`temp` float NOT NULL,
`salinity` float DEFAULT NULL,
`pH` float DEFAULT NULL,
`ammonia` float DEFAULT NULL,
`nitrites` float DEFAULT NULL,
`nitrates` float DEFAULT NULL,
`reader` varchar(20) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Any help would be great to figure this out. I am still troubleshooting but wanted to reach out to the community for some input.