So I wanted to generate something more inline with what I am doing.
So all of my circles I am sending to ES are generated in X,Y,Z using a unit sphere.
To demonstrate this I wrote a python script for you that you can use / modify with elasticsearch 8.7. It handles the mappings and such so other than telling Kibana the index exists and time is what you are using, it should just work for you.
It will dump out to stdout the 2 polygons, the first loads fine but plots are wrong:
Here is the one that is considered valid, but plots wrong:
I normalize so that my coordinate system is 0-360deg for longitude (however the geo_point I was getting errors when I had it from 0-360, so I left it at -180,180).
I noticed in some cases my winding could be wrong, so I enforce it using geojson_rewind.
Here is the failing polygon:
{"time": "2023-05-23T00:05:00", "center_point": [1.25, 45.0], "poly": {"type": "Polygon", "coordinates": [[[0.866430833706147, 34.69353612407307], [0.5596083542616839, 34.69658776328635], [0.25284221113031435, 34.701383745964414], [359.9461574779588, 34.707924804887966], [359.6395792669232, 34.71621193919848], [359.33313274064017, 34.726246414299965], [359.026843124123, 34.73802976173303], [358.7207357167928, 34.75156377901997], [358.4148359045565, 34.76685052947927], [358.1091691719616, 34.783892342008244], [357.8037611144385, 34.80269181083146], [357.4986374506427, 34.82325179521299], [357.19382403490624, 34.84557541913001], [356.88934686981094, 34.86966607090475], [356.58523211889434, 34.895527402792204], [356.2815061195004, 34.92316333051972], [355.9781953957858, 34.95257803277539], [355.6753266718949, 34.98377595064078], [355.3729268853147, 35.01676178696428], [355.0710232004231, 35.051540505669934], [354.7696430222418, 35.08811733099728], [354.46881401040804, 35.12649774666669], [354.1685640933778, 35.16668749496457], [353.86892148287376, 35.20869257574249], [353.5699146885917, 35.25251924532366], [353.2715725331805, 35.29817401531018], [352.97392416750847, 35.34566365128352], [352.6769990862319, 35.39499517139072], [352.380827143681, 35.44617584480801], [352.0854385700784, 35.49921319007342], [351.7908639881056, 35.554114973278956], [351.4971344298352, 35.610889206112816], [351.20428135404427, 35.66954414374149], [350.9123366639272, 35.730088282520796], [350.62133272522436, 35.7925303575246], [350.33130238478714, 35.856879339879136], [350.04227898959516, 35.92314443389028], [349.75429640624776, 35.991335073950594], [349.46738904094735, 36.06146092121179], [349.18159185999696, 36.133531860008155], [348.8969404108311, 36.20755799401527], [348.6134708436035, 36.28354964212769], [348.3312199333522, 36.36151733403827], [348.05022510276626, 36.44147180550133], [347.7705244455767, 36.52342399326055], [347.4921567505969, 36.60738502962158], [347.2151615264363, 36.69336623664862], [346.939579026914, 36.781379119962956], [346.6654502771979, 36.871435362120344], [346.3928171006979, 36.963546815543204], [346.12172214673876, 37.057725494982265], [345.8522089190438, 37.153983569481014], [345.58432180505736, 37.252333353815324], [345.31810610613655, 37.352787299378925], [345.05360806864365, 37.455357984484415], [344.79087491597085, 37.5600581040477], [344.5299548815295, 37.666900458622564], [344.2708972427383, 37.775897942750404], [344.0137523560431, 37.88706353258851], [343.7585716930048, 38.00041027277884], [343.5054078774897, 38.115951262517285], [343.2543147239994, 38.23369964078163], [343.00534727717724, 38.35366857067507], [342.7585618525281, 38.475871222839466], [342.514016078391, 38.600320757891154], [342.27176893920216, 38.72703030783011], [342.0318808200881, 38.85601295637062], [341.79441355282836, 38.98728171814032], [341.55943046322733, 39.12084951669152], [341.32699641993474, 39.256729161266854], [341.0971778847556, 39.39493332225912], [340.87004296448794, 39.53547450530229], [340.64566146432867, 39.678365023928876], [340.4241049428856, 39.82361697072602], [340.20544676883395, 39.97124218692018], [339.9897621792542, 40.121252230318056], [339.77712833968627, 40.273658341528346], [339.5676244059345, 40.4284714083869], [339.3613315876552, 40.585701928504854], [339.158333213755, 40.74535996985683], [338.958714799628, 40.90745512932376], [338.762564116253, 41.071996489102226], [338.56997126117017, 41.238992570889906], [338.3810287313517, 41.408451287753714], [338.19583149797376, 41.58037989358539], [338.0144770830929, 41.75478493004663], [337.83706563822193, 41.93167217090373], [337.66370002479124, 42.11104656364932], [337.4944858964743, 42.29291216830745], [337.329531783343, 42.477272093315946], [337.16894917780894, 42.664128428378724], [337.0128526222914, 42.85348217417944], [336.86135979853907, 43.04533316884698], [336.7145916185146, 43.239680011062404], [336.57267231673285, 43.43651997969731], [336.4357295439221, 43.63584894987318], [336.30389446185393, 43.83766130533247], [336.17730183916154, 44.04194984701362], [336.0560901479361, 44.248705697723786], [335.94040166086006, 44.45791820280614], [335.8303825485979, 44.669574826702245], [335.7261829771278, 44.88366104531423], [335.62795720465147, 45.100160234077165], [335.53586367767286, 45.31905355165888], [335.45006512578124, 45.54031981921202], [335.37072865461784, 45.76393539511291], [335.2980258364384, 45.989874045132346], [335.23213279761575, 46.21810680799637], [335.17323030234763, 46.44860185630898], [335.1215038317524, 46.68132435282627], [335.07714365744175, 46.916236302089125], [335.0403449085608, 47.153296397443796], [335.01130763117845, 47.39245986350343], [334.99023683879017, 47.6336782941302], [334.97734255257296, 47.87689948604853], [334.97283982989194, 48.12206726823311], [334.9769487794127, 48.36912132725297], [334.9898945610161, 48.61799702879463], [335.011907368544, 48.86862523563314], [335.0432223932244, 49.12093212237054], [335.0840797654378, 49.374838987317716], [335.13472447228315, 49.63026206195575], [335.1954062481955, 49.88711231848178], [335.2663794356464, 50.14529527601582], [335.347902812733, 50.40471080612654], [335.4402393842288, 50.66525293841942], [335.543656132431, 50.92680966702595], [335.6584237239035, 51.18926275893233], [335.78481616797376, 51.45248756519687], [335.9231104226133, 51.71635283622071], [336.07358594311034, 51.980720542362285], [336.23652416873557, 52.24544570131778], [336.4122079424227, 52.51037621383077], [336.6009208583326, 52.775352709441144], [336.8029465320571, 53.04020840413812], [337.01856778815886, 53.30476897194236], [337.24806575974145, 53.568852432607066], [337.4917188948207, 53.83226905779681], [337.749801864431, 54.09482129827268], [338.0225843676725, 54.35630373478254], [338.3103298292931, 54.61650305552088], [338.6132939859373, 54.87519806318313], [338.93172335787904, 55.132159714788735], [339.26585360393307, 55.3871511975818], [339.6159077582998, 55.63992804443332], [339.9820943493884, 55.890238292257926], [340.3646054021727, 56.13782268701533], [340.76361432739986, 56.38241493888449], [341.17927370298463, 56.62374203116916], [341.6117129552027, 56.86152458640977], [342.0610359498299, 57.09547729302814], [342.52731850615976, 57.32530939561137], [343.01060584984344, 57.55072525163998], [343.51091002370674, 57.77142495707429], [344.0282072790595, 57.98710504272547], [344.562435473469, 58.19745924274699], [345.1134915044421, 58.402179335882956], [345.68122881186315, 58.600956059301375], [346.2654549852644, 58.793480093921126], [346.8659295149297, 58.979443119116006], [347.4823617283381, 59.15853893355518], [348.11440895538027, 59.330464637729094], [348.7616749669951, 59.49492187242768], [349.42370873222444, 59.65161810610933], [350.1000035380352, 59.80026796274674], [350.78999651448953, 59.940594580393835], [351.493068604853, 60.07233098942182], [352.2085450159587, 60.19522149815874], [352.93569617856235, 60.30902307258184], [353.67373924055573, 60.41350669579386], [354.42184010783205, 60.50845869230694], [355.1791160384488, 60.593682001701694], [355.9446387857097, 60.66899738605511], [356.7174382751335, 60.734244555672085], [357.4965067892865, 60.78928319812665], [358.28080362347566, 60.83399389643268], [359.0692601646763, 60.868278923316325], [359.8607853361847, 60.89206290003913], [0.6542713416947095, 60.90529330999572], [1.4485996351365316, 60.90794085934353], [2.2426470369687195, 60.89999967916492], [3.035291913909248, 60.88148736605587], [3.8254203374747817, 60.852444860516286], [4.611932137239023, 60.81293616501333], [5.393746767394305, 60.76304790603597], [6.169808909900837, 60.702888746783906], [6.93909374404501, 60.63258865928047], [7.700611820341976, 60.55229806660743], [8.453413486100885, 60.46218686759066], [9.196592820269075, 60.36244335758248], [9.929291046018818, 60.25327305997009], [10.65069940057208, 60.13489748368275], [11.360061452616947, 60.0075528222731], [12.056674868046798, 59.871488610130385], [12.73989263436357, 59.72696635106285], [13.409123762720299, 59.57425813390103], [14.063833494060873, 59.41364524895557], [14.70354304204676, 59.24541681815591], [15.327828910387893, 59.069868450541854], [15.936321825820698, 58.88730093351997], [16.528705330344224, 58.69801896897295], [17.10471407752226, 58.5023299619588], [17.664131877789714, 58.30054286839245], [18.20678953690225, 58.0929671067947], [18.73256253008043, 57.87991153794411], [19.24136855216244, 57.66168351509879], [19.733164981351706, 57.438588006379135], [20.20794629104779, 57.210926789929644], [20.665741440924194, 56.97899772161352], [21.106611274962347, 56.743094074240254], [21.53064595067883, 56.50350394668202], [21.937962420364443, 56.26050974069605], [22.328701981862935, 56.014387702831755], [22.70302791330039, 55.76540752845633], [23.06112320327503, 55.51383202467274], [23.403188385355747, 55.259916828720755], [23.729439483329543, 55.00391017833713], [24.040106071492232, 54.74605273049482], [24.335429452388382, 54.48657742493703], [24.61566095276919, 54.22570938896005], [24.88106033714189, 53.963665879972694], [25.131894337107553, 53.70065626246324], [25.36843529372146, 53.43688201613001], [25.59095990932815, 53.17253677207467], [25.7997481047164, 52.907806374112326], [25.995081976976792, 52.64286896241628], [26.177244853118395, 52.377895076883476], [26.346520434283946, 52.113047777777204], [26.503192025286353, 51.848482781372795], [26.64754184415318, 51.58434860849965], [26.779850406398623, 51.32078674403526], [26.900395978829124, 51.057931805564586], [27.009454097824687, 50.795911719569006], [27.10729714720202, 50.53484790365299], [27.19419399096455, 50.27485545345345], [27.270409656453978, 50.01604333300476], [27.336205063648322, 49.75851456745388], [27.39183679658379, 49.502366437132586], [27.4375569131181, 49.24769067209921], [27.47361278948864, 48.99457364635976], [27.500246996356736, 48.74309657106853], [27.517697203256375, 48.49333568609204], [27.526196108590966, 48.245362449396], [27.525971392533506, 47.99924372378587], [27.51724569039277, 47.75504196059538], [27.500236584200707, 47.5128153799761], [27.475156610461795, 47.272618147494555], [27.44221328217793, 47.03450054679121], [27.401609123425374, 46.79850914810017], [27.353541714912467, 46.56468697246725], [27.298203749088202, 46.33307365153996], [27.235783093504324, 46.10370558283471], [27.166462861254672, 45.876616080415204], [27.090421487429126, 45.65183552094125], [27.007832810623142, 45.42939148507016], [26.9188661586399, 45.20930889421239], [26.823686437609467, 44.991610142661486], [26.72245422383122, 44.776315225133004], [26.61532585771755, 44.5634418597617], [26.50245353928642, 44.35300560661655], [26.38398542471009, 44.145019981804936], [26.260065723483876, 43.93949656724441], [26.130834795829855, 43.73644511618897], [25.996429249996027, 43.535873654601936], [25.85698203915365, 43.3377885784733], [25.71262255763253, 43.14219474718276], [25.56347673626965, 42.9490955730132], [25.409667136675523, 42.75849310692198], [25.251313044251503, 42.5703881206783], [25.088530559816377, 42.38478018547676], [24.921432689721826, 42.201667747137314], [24.750129434357518, 42.02104819800206], [24.5747278749638, 41.84291794563849], [24.39533225868672, 41.667272478458734], [24.212044081823024, 41.494106428362464], [24.02496217121717, 41.32341363051041], [23.834182763782053, 41.155187180333144], [23.639799584126308, 40.98941948787874], [23.44190392027849, 40.826102329599905], [23.24058469750804, 40.66522689768007], [23.035928550247036, 40.50678384699479], [22.828019892125553, 40.35076333980291], [22.616940984135795, 40.197155088259265], [22.40277200094721, 40.04594839483836], [22.185591095395637, 39.89713219075565], [21.965474461175802, 39.75069507247072], [21.742496393766828, 39.606625336353964], [21.51672934962403, 39.46491101159567], [21.288244003671252, 39.325539891433955], [21.057109305130894, 39.18849956277555], [20.82339253172745, 39.05377743428037], [20.5871593423052, 38.92136076297917], [20.348473827896328, 38.791236679489984], [20.107398561281798, 38.663392211897786], [19.863994645082187, 38.5378143083586], [19.618321758419995, 38.41448985848717], [19.37043820219276, 38.293405713585344], [19.120400942996184, 38.17454870576548], [18.868265655736877, 38.057905666021774], [18.614086764974104, 37.94346344129962], [18.357917485027997, 37.83120891061141], [18.09980985889291, 37.72112900024533], [17.839814795993277, 37.61321069811131], [17.57798210881782, 37.50744106726701], [17.314360548469494, 37.40380725866434], [17.048997839164713, 37.30229652315579], [16.781940711718107, 37.202896222797875], [16.513234936045308, 37.10559384148726], [16.242925352717577, 37.01037699496405], [15.971055903599733, 36.91723344021456], [15.697669661603754, 36.82615108430514], [15.422808859587747, 36.73711799267645], [15.14651491843091, 36.650122396927145], [14.868828474312977, 36.565152702113615], [14.589789405226668, 36.48219749359218], [14.30943685675112, 36.40124554342813], [14.027809267111877, 36.32228581639537], [13.744944391555293, 36.24530747558918], [13.460879326060535, 36.170299887673345], [13.175650530415794, 36.09725262778226], [12.8892938506811, 36.02615548409733], [12.601844541062178, 35.95699846211633], [12.313337285216676, 35.889771788632935], [12.023806217015874, 35.82446591544391], [11.733284940782596, 35.76107152279909], [11.441806551025934, 35.69957952260989], [11.149403651693433, 35.63998106143039], [10.85610837495949, 35.58226752322499], [10.561952399570146, 35.52643053193531], [10.266966968761551, 35.47246195385881], [9.97118290777081, 35.4203538998509], [9.67463064095665, 35.370098727361494], [9.377340208546798, 35.32168904231643], [9.079341283028441, 35.27511770085406], [8.78066318519825, 35.230377810925944], [8.481334899888054, 35.18746273377091], [8.181385091380037, 35.14636608527069], [7.880842118528506, 35.10708173719517], [7.579734049600802, 35.06960381834463], [7.278088676853258, 35.03392671559611], [6.975933530854775, 35.00004507486043], [6.673295894572561, 34.96795380195624], [6.370202817233007, 34.93764806340674], [6.066681127970753, 34.909123287164725], [5.7627574492784674, 34.88237516327083], [5.458458210270408, 34.85739964444999], [5.153809659771468, 34.834192946650404], [4.8488378792444, 34.81275154952897], [4.543568795566443, 34.7930721968873], [4.23802819366756, 34.77515189706143], [3.9322417290413796, 34.75898792326871], [3.6262349401405913, 34.74457781391457], [3.3200332606678558, 34.73191937286184], [3.0136620317728102, 34.72101066966497], [2.707146514167448, 34.711850039771285], [2.400511900168965, 34.704436084690975], [2.0937833256823524, 34.698767672137286], [1.7869858821323987, 34.694843936138554], [1.4801446283568112, 34.69266427712282], [1.1732846024694936, 34.69222836197581], [0.866430833706147, 34.69353612407307]]]}}
import math
import json
import socket
import numpy as np
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch, helpers
from elasticsearch.helpers import BulkIndexError
from geojson_rewind import rewind
def lla2ecef(lat, lon, alt):
# WGS-84 ellipsoid parameters
a = 6378137 # semi-major axis (equatorial radius) in meters
f = 1 / 298.257223563 # flattening
b = a * (1 - f) # semi-minor axis (polar radius) in meters
# convert lat, lon to radians
lat_rad = math.radians(lat)
lon_rad = math.radians(lon)
# Prime vertical radius of curvature at the specific latitude
N = a / math.sqrt(1 - ((2*f - f**2) * math.sin(lat_rad)**2))
# Calculate ECEF coordinates
x = (N + alt) * math.cos(lat_rad) * math.cos(lon_rad)
y = (N + alt) * math.cos(lat_rad) * math.sin(lon_rad)
z = (b**2/a**2 * N + alt) * math.sin(lat_rad)
return x, y, z
def ecef2lla(x: float, y: float, z: float) -> tuple[float, float, float]:
"""
Convert ECEF coordinates to LLA coordinates.
Args:
x: X coordinate in meters.
y: Y coordinate in meters.
z: Z coordinate in meters.
Returns:
Latitude in degrees.
Longitude in degrees.
Altitude in meters.
"""
# Earth constants
a = 6378137 # Semi-major axis of the Earth in meters.
e = 8.181919084261345e-2 # Eccentricity of the Earth.
# Calculate the radius vector.
r = np.sqrt(x ** 2 + y ** 2)
# Calculate the eccentricity squared.
e_sq = e ** 2
# Calculate the flattening of the Earth.
f = 1 / 298.257223563
# Calculate the semi-minor axis of the Earth in meters.
b = a * (1 - f)
# Calculate the eccentricity squared of the prime vertical.
ep_sq = (a ** 2 - b ** 2) / b ** 2
# Calculate the second eccentricity squared.
ee = a ** 2 - b ** 2
# Calculate the auxiliary quantity f.
f = 54 * b ** 2 * z ** 2
# Calculate the auxiliary quantity g.
g = r ** 2 + (1 - e_sq) * z ** 2 - e_sq * ee
# Calculate the auxiliary quantity c.
c = e_sq ** 2 * f * r ** 2 / g ** 3
# Calculate the auxiliary quantity s.
s = (1 + c + np.sqrt(c ** 2 + 2 * c)) ** (1 / 3.)
# Calculate the auxiliary quantity p.
p = f / (3. * (g ** 2) * (s + (1. / s) + 1) ** 2)
# Calculate the auxiliary quantity q.
q = np.sqrt(1 + 2 * p * e_sq ** 2)
# Calculate the reduced semi-major axis.
r_0 = -(p * e_sq * r) / (1 + q) + np.sqrt(0.5 * (a ** 2) * (1 + (1. / q))
- p * (z ** 2) * (1 - e_sq) / (q * (1 + q)) - 0.5 * p * (r ** 2))
# Calculate the distance to the prime vertical.
u = np.sqrt((r - e_sq * r_0) ** 2 + z ** 2)
# Calculate the distance to the ellipsoid.
v = np.sqrt((r - e_sq * r_0) ** 2 + (1 - e_sq) * z ** 2)
# Calculate the height above the ellipsoid.
z_0 = (b ** 2) * z / (a * v)
# Calculate the altitude.
h = u * (1 - b ** 2 / (a * v))
# Calculate the latitude.
phi = np.arctan((z + ep_sq * z_0) / r)
# Calculate the longitude.
lambd = np.arctan2(y, x)
# Return the latitude, longitude, and altitude.
return phi * 180 / np.pi, lambd * 180 / np.pi, h
def create_index_with_mapping(es_client, index_name):
"""
Create Elasticsearch index with the given mapping.
:param es: Elasticsearch instance
:param index_name: Name of the index to create
"""
index_mapping = {
"time": {
"type": "date",
"format": "strict_date_optional_time||epoch_second"
},
"center_point": {
"type": "geo_point"
},
"poly": {
"type": "geo_shape",
}
}
es_client.options(ignore_status=[400]).indices.create(index=index_name)
es_client.indices.put_mapping(index=index_name, properties=index_mapping)
# Function to generate a circle's points in ECEF and convert to LLA
def generate_circle_ecef_to_lla(x_center, y_center, z_center, radius, num_points):
circle_points = []
for i in range(num_points):
theta = 2 * math.pi * i / num_points
x = x_center + radius * math.cos(theta)
y = y_center + radius * math.sin(theta)
z = z_center
lat, lon, _ = ecef2lla(x, y, z)
# Map between 0 and 360.0
lon = np.mod( lon + 360.0, 360.0 )
circle_points.append([lon, lat]) # Adjusted for [lon, lat] order
# Repeat the first point to close the polygon
circle_points.append(circle_points[0])
return circle_points
def yield_circles(radius, lat, sides):
#for ii in range(121, 124):
for ii in range(2):
circle_time = start_time + timedelta(minutes=5*ii)
# 288 circles in 1 day
lon = ii*1.25
x, y, z = lla2ecef(lat, lon, 0.0)
points = generate_circle_ecef_to_lla(x, y, z, radius, sides)
circle = {}
circle["time"] = circle_time.isoformat()
# This has to be between (-180,180)
circle["center_point"] = ((lon+180.0) % 360.0 - 180.0, lat)
circle["poly"] = {
'type': 'Polygon',
'coordinates': [points]}
inputval = {"geometry": circle["poly"], "type": "Feature"}
poly_rw = rewind(inputval)
circle["poly"] = poly_rw["geometry"]
print(json.dumps(circle))
yield circle
# Hard coded, but change as your heart desires
radius = 2000e3
sides = 360
lat = 45.0
index_name = "circles"
start_time = datetime.combine(datetime.today().date(), datetime.min.time())
es_host = f"{socket.gethostname()}"
es_port = 9200
es_client = Elasticsearch(f"http://{es_host}:{es_port}", request_timeout=30.0)
create_index_with_mapping(es_client, index_name)
data = ({
"_index": index_name,
"_source": es_record,
} for es_record in yield_circles(radius, lat, sides))
try:
success, errors = helpers.bulk(es_client, data, raise_on_error=False)
# success, errors = helpers.bulk(es_client, data, raise_on_error=False)
except BulkIndexError as e:
print(f"{e}")
print(json.dumps(errors))
