Bulk upload documents to Site Search from a JSON file

Elastic Search
1

I just wanted to share some code I helped put together for a customer. They had a JSON document of records that they wanted to upload in bulk to Site Search via our API.

books.json

[
  {
    "id": "1",
    "title": "Tom Sawyer",
    "pages": 200
  },
  {
    "id": "2",
    "title": "Catcher in the Rye",
    "pages": 300
  }
]

The first challenge was converting their JSON file to a format that Site Search requires. We came up with the following:

convert.js

var FILE_NAME = "books.json";

var isValidNumber = num => {
  return !isNaN(num);
};

var fs = require("fs");
var obj = JSON.parse(fs.readFileSync(FILE_NAME, "utf8"));
var convert = o => {
  return Object.entries(o).reduce(
    (p, [k, v]) => {
      if (k === "id") {
        return {
          external_id: v,
          ...p
        };
      }
      return {
        ...p,
        fields: p.fields.concat({
          name: k,
          value: v,
          type: isValidNumber(v) ? "number" : "string"
        })
      };
    },
    { fields: [] }
  );
};
var updated = obj.map(convert);

fs.writeFile("updated.json", JSON.stringify(updated), err => {
  if (err) throw err;
});

Assuming that convert.js is located in the same directory as books.json, you would run:

node convert.js

This produces a file in the correct format for Site Search:

updated.json

[
  {
    "external_id": "1",
    "fields": [
      { "name": "title", "value": "Tom Sawyer", "type": "string" },
      { "name": "pages", "value": 200, "type": "number" }
    ]
  },
  {
    "external_id": "2",
    "fields": [
      { "name": "title", "value": "Catcher in the Rye", "type": "string" },
      { "name": "pages", "value": 300, "type": "number" }
    ]
  }
]

We then created another file in the same directory, and configured our API key, document type, and engine name:

upload.js

var ENGINE_NAME = "book-engine";
var DOCUMENT_TYPE = "books";
var API_KEY = "{YOUR_KEY_HERE}";

var fs = require("fs");
var json = JSON.parse(fs.readFileSync("updated.json", "utf8"));

var SiteSearchClient = require("@elastic/site-search-node");
var client = new SiteSearchClient({
  apiKey: API_KEY
});

client.documents.batchCreate(
  {
    engine: ENGINE_NAME,
    documentType: DOCUMENT_TYPE
  },
  json,
  100,
  (e, o) => {
    console.log(e);
    console.log(o);
  }
);

We then installed the site search node client, and then ran our script:

npm install @elastic/site-search-node
node upload.js

And voilà, our documents have now been indexed.

1 Like
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.