DEV Community

karthik22061993
karthik22061993

Posted on

How to create/ update large quantity records in azure cosmos database

I have 1500 records to be created in azure cosmos database, I just loop through using Javascript code. I have REST API connection to the database. I feed the data to be updated as JSON array.

The problem is when I pass entire data, azure database timesout or send ECONNECTIONRESET. Many of you would create huge amount of records in the database and there is might be some efficient way to overcome this problem. I want your suggestion.

Since majority of records is not found, many times create new record part is hit and I have never pushed such huge amount of data before. Any suggestion or new idea will really help me.

Note : I run this javascript code using mocha unit test

Below is code snippet

Record.js

const fetch = require('node-fetch');
let connectionAzureDataBase = "abc...";
let SubscriptionKey = "xyz";
let promises = [];
let j = -1;

module.exports = {
checkRecord
}
function checkRecord (req) {
 for (let i = 0; i < req.body.length; i++) {
    promises[j] = new Promise(async function(resolve, reject) {
      //check if record exist in azure
      var apiUrl = APICheckRecord( req.body[i].recordName);
            fetch(apiUrl , { headers:connectionAzureDataBase})
            .then(res => res.json())
            .then(record => {
              if(record) {
                console.log("Record Found");
              } else {
                console.log("Record not Found, calling API to create Record");
                var apiUrl = APICreateNewRecord( req.body[i].recordName);
                fetch(apiUrl , { headers:connectionAzureDataBase})
                .then(res => res.json())
                .then(recordCreated => {
                  if(recordCreated) {
                   console.log("record created successfully");
                   resolve("record created successfully");
                  } else {
                   console.log("Encountered some unexpected condition");
                   resolve("Encountered some unexpected condition");
                  }
                 }).catch(err => {
                     console.log("record could not be created");
                      resolve("record could not be created");
                 }) 
              }
             }).catch(err => {
                console.log("record not found");
                resolve("record not found");
             })
     })// close promise

 }// close for

 let replies = await Promise.all(promises); 
 let promise1 = new Promise (function(resolve,reject) {
   resolve(replies);
 }) 
}


Record.spec.js

const Records = require("Record.js);

 it("should find/create records", async function() {
        this.timeout(6000000);
        try { 
         let req =[
          {
            "recordName": "Xyz",
            "recordDate": "12-06-2020"
          },
          {
            "recordName": "Abc",
            "recordDate": "13-06-2020"
           }
          ]
          let reply = await Records.checkRecord(req);
          console.log(JSON.stringify(reply));

        } catch(err) {
            console.log(err);
        }     
    })

Error

  message: 'request to https://apim-dev.azure-api.net/api/portal/records/?recordName="Xyz" failed, reason: read ECONNRESET',
  type: 'system',
  errno: 'ECONNRESET',
  code: 'ECONNRESET' }

message: 'request to https://apim-dev.azure-api.net/api/portal/createRecords/ failed, reason: read ECONNRESET',
  type: 'system',
  errno: 'ECONNRESET',
  code: 'ECONNRESET' }

This is sample of data that is passed, I have 1500 such records
SampleData

[
 {
   "recordName": "Xyz",
    "recordDate": "12-06-2020"
 },
{
   "recordName": "Abc",
    "recordDate": "13-06-2020"
 }
]

Top comments (0)