So having gotten further by doing a build of the 8.0.0-SNAPSHOT from source, I now have a new issue.
I am able to create data frames and print schemas. However, every query returns null for every field. Here is my code and the schemas:
val dsParams = Map(
"es.nodes.wan.only" -> "true",
"es.port" -> "9201",
"es.net.ssl" -> "true",
"es.net.ssl.cert.allow.self.signed" -> "true",
"es.net.ssl.truststore.location" -> "file:///etc/pki/java/truststore.jks",
"es.net.ssl.truststore.type" -> "JKS",
"es.net.ssl.truststore.pass"-> "changeit",
"es.net.ssl.keystore.location" -> "file:///etc/pki/java/keystore.jks",
"es.net.ssl.keystore.type" -> "jks",
"es.net.ssl.keystore.pass"-> "changeit",
"es.clustername" -> "BDP_oedl.dev.oedl.tradoc.army.mil",
"es.nodes" -> "oedevnode06.dev.oedl.tradoc.army.mil"
)
// Create DataFrame using the "es" format
val docFrame = spark.read.format("es").options(dsParams).load("coalesce-oedocument")
docFrame.createOrReplaceTempView("esdocview")
// Create DataFrame using the "es" format
val eventFrame = spark.read.format("es").options(dsParams).load("coalesce-oeevent")
eventFrame.createOrReplaceTempView("eseventview")
docFrame.printSchema()
eventFrame.printSchema()
val esdocs = spark.sql("select recordset.coalesceentity.objectkey from esdocview limit 20")
esdocs.show()
val esevents = spark.sql("select recordset.coalesceentity.objectkey from eseventview limit 20")
esdocs.show()
root
|-- recordset: struct (nullable = true)
| |-- coalesceentity: struct (nullable = true)
| | |-- datecreated: timestamp (nullable = true)
| | |-- entityid: string (nullable = true)
| | |-- entityidtype: string (nullable = true)
| | |-- lastmodified: timestamp (nullable = true)
| | |-- name: string (nullable = true)
| | |-- objectkey: string (nullable = true)
| | |-- source: string (nullable = true)
| | |-- status: string (nullable = true)
| | |-- title: string (nullable = true)
| | |-- version: string (nullable = true)
| |-- oedocumentrecordset: struct (nullable = true)
| | |-- categories: string (nullable = true)
| | |-- content: string (nullable = true)
| | |-- contentlength: long (nullable = true)
| | |-- datasource: string (nullable = true)
| | |-- dateingested: timestamp (nullable = true)
| | |-- documentdate: timestamp (nullable = true)
| | |-- documentlastmodifieddate: timestamp (nullable = true)
| | |-- documentsource: string (nullable = true)
| | |-- documenttitle: string (nullable = true)
| | |-- documenttype: string (nullable = true)
| | |-- issimulation: boolean (nullable = true)
| | |-- md5sum: string (nullable = true)
| | |-- ner_date: string (nullable = true)
| | |-- ner_location: string (nullable = true)
| | |-- ner_money: string (nullable = true)
| | |-- ner_organization: string (nullable = true)
| | |-- ner_percent: string (nullable = true)
| | |-- ner_person: string (nullable = true)
| | |-- ner_time: string (nullable = true)
| | |-- ontologyreference: string (nullable = true)
| | |-- pmesiipteconomic: float (nullable = true)
| | |-- pmesiiptinformation: float (nullable = true)
| | |-- pmesiiptinfrastructure: float (nullable = true)
| | |-- pmesiiptmilitary: float (nullable = true)
| | |-- pmesiiptphysicalenvironment: float (nullable = true)
| | |-- pmesiiptpolitical: float (nullable = true)
| | |-- pmesiiptsocial: float (nullable = true)
| | |-- pmesiipttime: float (nullable = true)
| | |-- sourceuri: string (nullable = true)
| | |-- tags: string (nullable = true)
| | |-- wordcount: long (nullable = true)root
|-- recordset: struct (nullable = true)
| |-- coalesceentity: struct (nullable = true)
| | |-- datecreated: timestamp (nullable = true)
| | |-- entityid: string (nullable = true)
| | |-- entityidtype: string (nullable = true)
| | |-- lastmodified: timestamp (nullable = true)
| | |-- name: string (nullable = true)
| | |-- objectkey: string (nullable = true)
| | |-- source: string (nullable = true)
| | |-- status: string (nullable = true)
| | |-- title: string (nullable = true)
| | |-- version: string (nullable = true)
| |-- eventrecordset: struct (nullable = true)
| | |-- actiongeoadm1code: string (nullable = true)
| | |-- actiongeoadm2code: string (nullable = true)
| | |-- actiongeocountrycode: string (nullable = true)
| | |-- actiongeofeatureid: string (nullable = true)
| | |-- actiongeofullname: string (nullable = true)
| | |-- actiongeolocation: struct (nullable = true)
| | | |-- lat: double (nullable = true)
| | | |-- lon: double (nullable = true)
| | |-- actiongeotype: integer (nullable = true)
| | |-- avgtone: float (nullable = true)
| | |-- datasource: string (nullable = true)
| | |-- dateadded: integer (nullable = true)
| | |-- datetime: timestamp (nullable = true)
| | |-- day: integer (nullable = true)
| | |-- eventbasecode: string (nullable = true)
| | |-- eventcode: string (nullable = true)
| | |-- eventrootcode: string (nullable = true)
| | |-- fractiondate: float (nullable = true)
| | |-- globaleventid: integer (nullable = true)
| | |-- goldsteinscale: float (nullable = true)
| | |-- isrootevent: integer (nullable = true)
| | |-- issimulation: boolean (nullable = true)
| | |-- monthyear: integer (nullable = true)
| | |-- numarticles: integer (nullable = true)
| | |-- nummentions: integer (nullable = true)
| | |-- numsources: integer (nullable = true)
| | |-- ontologyreference: string (nullable = true)
| | |-- pmesiipteconomic: float (nullable = true)
| | |-- pmesiiptinformation: float (nullable = true)
| | |-- pmesiiptinfrastructure: float (nullable = true)
| | |-- pmesiiptmilitary: float (nullable = true)
| | |-- pmesiiptphysicalenvironment: float (nullable = true)
| | |-- pmesiiptpolitical: float (nullable = true)
| | |-- pmesiiptsocial: float (nullable = true)
| | |-- pmesiipttime: float (nullable = true)
| | |-- quadclass: integer (nullable = true)
| | |-- sourceurl: string (nullable = true)
| | |-- tags: string (nullable = true)
| | |-- year: integer (nullable = true)
Both queries return nulls.
I am using Elastic 5.4.3 . All works fine in Kibana and java.