Hi,
Actually, I am a beginner at elastic. I want to use elastic for a music app so we can assume all the possible entities of a music app i.e Spotify. I have a doubt that can we populate from the second index (assume=> artist ) whose reference id is stored in the first index(assume =>Song Release). Like in MongoDb we have references concepts and we can populate with $lookup with a single query or should I use multiple queries on different indexes and I am not sure that how can we do this or should I store all data in one index but that would be a pain if we want to update something.
Welcome!
I'd totally forget about the existing model and would just think about the usage to build the right "search" objects for my use case.
Here I can't really answer as I don't know the use case.
Basically I'd recommend to ask yourself 2 questions:
- What kind of objects my users want to get back as a response? If it's object
Song, then just index objectSong - What typical attributes my users want to search for? Let say I need attribute
artist,albumandgenre, just index those attributes within objectSongwhatever the original source of those attributes is.
HTH
var ArtistSchema = new Schema({
name: {
type: String,
trim: true,
},
bio: {
type: String,
},
image: {
type: String,
trim: true,
},
followers: {
type: Number,
default:0
},
startDate: {
type: Date,
},
endDate: {
type: Date,
},
isDeleted: {
type: Boolean,
default: false,
}
});
"use strict";
import mongoose from "mongoose";
import mongoosastic from "mongoosastic";
const Schema = mongoose.Schema;
let SongSchema = new Schema(
{
title: {
type: String,
},
artistId: [{
type: Schema.Types.ObjectId,
ref: 'Artist'
}],
genreId: [{
type: Schema.Types.ObjectId,
ref: 'Genre'
}],
timeLength: {
type: Number,
},
listenCount: {
type: Number,
default:0
},
image: {
type: String,
},
songUrl:{
type:String
},
isDeleted: {
type: Boolean,
default: false,
}
},
{ timestamps: true }
);
SongSchema.plugin(mongoosastic, {
hosts: [
'localhost:9200'
]
});
let GenreSchema = new Schema(
{
genreName: {
type: String,
es_indexed:true
},
isDeleted: {
type: Boolean,
default: false,
}
},
{ timestamps: true }
);
let ReleaseSchema = new Schema(
{
name: {
type: String,
es_indexed: true,
},
artistId: [{
type: Schema.Types.ObjectId,
ref: 'Artist',
es_schema: ArtistSchema,
es_indexed:true,
es_select: 'name bio'
}],
genreId: [{
type: Schema.Types.ObjectId,
ref: 'Genre',
es_schema: GenreSchema,
es_indexed:true,
es_select: 'genreName'
}],
songId: [{
type: Schema.Types.ObjectId,
ref: 'Song',
es_schema: SongSchema,
es_indexed:true,
}],
type: {
type: String,
enum: [
appConstants.type.single,
appConstants.type.album
],
es_indexed: true,
default: appConstants.type.single,
},
image: {
type: String,
es_indexed: true,
},
isDeleted: {
type: Boolean,
es_indexed: true,
default: false,
}
},
{ timestamps: true }
);
ReleaseSchema.plugin(mongoosastic, {
hosts: [
'localhost:9200'
]
});
ReleaseSchema.plugin(mongoosastic, {
populate: [
{path: 'genreId', select: 'genreName'},
{path: 'artistId', select: 'name bio'},
{path: 'songId', select: 'title timeLength listenCount image songUrl'}
]
})
module.exports = mongoose.model("Release", ReleaseSchema, "releases");
{
"_index": "releases",
"_type": "_doc",
"_id": "61024c43f5ef9f2c5c870e12",
"_version": 2,
"_score": 0,
"fields": {
"songId.songUrl.keyword": [
"https://bishaan.s3.amazonaws.com/1626872596216Baca%20%20%20jireenya%20foon%20kootiif.mp3",
"https://bishaan.s3.amazonaws.com/1626872596216Baca%20%20%20jireenya%20foon%20kootiif.mp3"
],
"image": [
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png"
],
"genreId.genreName": [
"ROCKROCK",
"CLASSIC",
"POP"
],
"songId.listenCount": [
0,
0
],
"name.keyword": [
"ghjk"
],
"songId._id": [
"60f9411a31a4f70bf4424a07",
"60f9413b31a4f70bf4424a09"
],
"type": [
"album"
],
"artistId.bio.keyword": [
"hello bio here",
"hey there",
"yup"
],
"artistId.name.keyword": [
"Artist1",
"Artist 2",
"Artitst 3"
],
"songId.image": [
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png",
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png"
],
"songId.title.keyword": [
"GOAT",
"CHANDIGARH"
],
"isDeleted": [
false
],
"image.keyword": [
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png"
],
"type.keyword": [
"album"
],
"songId._id.keyword": [
"60f9411a31a4f70bf4424a07",
"60f9413b31a4f70bf4424a09"
],
"name": [
"ghjk"
],
"artistId.name": [
"Artist1",
"Artist 2",
"Artitst 3"
],
"songId.image.keyword": [
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png",
"https://bishaan.s3.us-east-2.amazonaws.com/16268689944891626867642041insta%20%283%29.png"
],
"songId.title": [
"GOAT",
"CHANDIGARH"
],
"songId.timeLength": [
303,
303
],
"songId.songUrl": [
"https://bishaan.s3.amazonaws.com/1626872596216Baca%20%20%20jireenya%20foon%20kootiif.mp3",
"https://bishaan.s3.amazonaws.com/1626872596216Baca%20%20%20jireenya%20foon%20kootiif.mp3"
],
"genreId.genreName.keyword": [
"ROCKROCK",
"CLASSIC",
"POP"
],
"artistId.bio": [
"hello bio here",
"hey there",
"yup"
]
}
}
**```**
**My schemas are here and the result returned from Kibana is above and I now I am facing a problem that if I update genre or artist or song that data is not getting updated in release index of elastic. Suppose I Update POP (Genre ) to POP1 or I change the name of artist1 to artist123 then it will be updated in artist schema but the data that exist in the release index that is artist1 but in artist index is updated.**
**The main query that I am not getting is how should I populate release data from elastic with the updated data.**
var process_output = function (data) {
console.log("data",data.hits.hits);
data = data.hits.hits;
let result = []
data.forEach(element => {
result.push(element._source)
});
console.log("PROCESS DATA",result);
return result;
}
let releases = await elastic_client.search({
index: 'releases'
})
// console.log("releases",releases);
releases = process_output(releases)
console.log("releases",releases);
releases.map((r)=>{
let artistId = r.artistId
artistId.map((a)=>{
console.log("a",a);
})
let genreId = r.genreId
genreId.map((a)=>{
console.log("genreId",a);
})
})Hi David Pilato, thanks for replying I have explained my problem in my new reply. It would be appreciated if you will take a look at that.
I think it would help if you recreate a minimal example which can be run from Kibana Dev Console as described in About the Elasticsearch category. It will help to better understand what you are doing. Please, try to keep the example as simple as possible.
A full reproduction script is something anyone can copy and paste in Kibana dev console, click on the run button to reproduce your use case. It will help readers to understand, reproduce and if needed fix your problem. It will also most likely help to get a faster answer.
Elasticsearch does not have anything like the lookup operator so you generally need to structure your data differently or run multiple queries.
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.