dojo.promiseAll with limit to process * promises at once

146
3
Jump to solution
03-09-2020 12:30 AM
Highlighted
New Contributor

Hi, I have a custom widget, which takes a csv file and updates it back with results from hosted layer. for example I have a 80k file, for which I should get the out fields by executing query from a hosted layer. 

1.when I use promiseAll with all 80k queryTask results in collection, it is getting stuck.

2.if I use layer.queryfeatures() one after other,too many requestes hitting the server and getting timeouts. Is there any way to limit the promises count to be taken at once? even if I had 80k promises in collection, if i can instruct system to take only 10 requests at a time that will solve my problem.

3.I have tried "reduce" to sequence the promises, even then once it hit layer.queryfeatures it is going asynchronous.

I am looking for some thing like async.eachLimit. can some one please help?

Reply
0 Kudos
1 Solution

Accepted Solutions
Highlighted
MVP Esteemed Contributor

Sandhya,

  Just create several arrays of queryTasks and queries.

var iterationCnt = 0;
var batchArray = [];
for(var i=0; i<csvItems.length; i=i+batchCount){
var finalQry = getQueryFromCSV(i, batchCount);
var query = new Query();
query.outFields = config.outFieldsList;
query.returnGeometry = false;
query.where = finalQry;
query.f = "json";

var queryTask = new QueryTask(config.HostedUrl);
//only if the array is 10 or less
if(iterationCnt >= 9){
//here we are pushing an object containing the queryTask and the query
//we are NOT executing the querytask yet
methodsCollection.push({qt:queryTask, q:query});
iterationCnt++;
}else{
//once the array is 10 then push to batchArray and methodsCollection is re-
//created as an empty array and count is reset to 0
batchArray.push(methodsCollection);
iterationCnt = 0;
methodsCollection = [];
methodsCollection.push({qt:queryTask, q:query});
}
}
//once all the csv items have been processed in batches of 10 then start executing the
//QueryTasks in those batches of 10 and do not more to the next batch until
//the first batch is done.
getBatchResults(0);

function getBatchResults(index) {
batchArray[index].map(function(qtIteration){
qtResults.push(qtIteration.qt.execute(qtIteration.q));
});
dojoPromiseAll(qtResults).then(function(results){
//do something with the results array
//move to next batch
if(index <= batchArray.length - 1){
getBatchResults(index++)
};
});
}‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

View solution in original post

3 Replies
Highlighted
New Contributor

Here is my code sample

const layer = new FeatureLayer(config.HostedUrl);
for(var i=0;i<csvItems.length;i=i+batchCount){

   var finalQry = getQueryFromCSV(i,batchCount);
   var query = new Query();
   query.outFields=config.outFieldsList;
   query.returnGeometry=false;
   query.where =finalQry;
   query.f="json";

   var queryTask = new QueryTask(config.HostedUrl);
   qryResults = queryTask.execute(query);

   methodsCollection.push(qryResults);
   //methodsCollection.push(layer.queryFeatures(query));
}

dojoPromiseAll(methodsCollection).then(promiseQueryFunction);

Reply
0 Kudos
Highlighted
MVP Esteemed Contributor

Sandhya,

  Just create several arrays of queryTasks and queries.

var iterationCnt = 0;
var batchArray = [];
for(var i=0; i<csvItems.length; i=i+batchCount){
var finalQry = getQueryFromCSV(i, batchCount);
var query = new Query();
query.outFields = config.outFieldsList;
query.returnGeometry = false;
query.where = finalQry;
query.f = "json";

var queryTask = new QueryTask(config.HostedUrl);
//only if the array is 10 or less
if(iterationCnt >= 9){
//here we are pushing an object containing the queryTask and the query
//we are NOT executing the querytask yet
methodsCollection.push({qt:queryTask, q:query});
iterationCnt++;
}else{
//once the array is 10 then push to batchArray and methodsCollection is re-
//created as an empty array and count is reset to 0
batchArray.push(methodsCollection);
iterationCnt = 0;
methodsCollection = [];
methodsCollection.push({qt:queryTask, q:query});
}
}
//once all the csv items have been processed in batches of 10 then start executing the
//QueryTasks in those batches of 10 and do not more to the next batch until
//the first batch is done.
getBatchResults(0);

function getBatchResults(index) {
batchArray[index].map(function(qtIteration){
qtResults.push(qtIteration.qt.execute(qtIteration.q));
});
dojoPromiseAll(qtResults).then(function(results){
//do something with the results array
//move to next batch
if(index <= batchArray.length - 1){
getBatchResults(index++)
};
});
}‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

View solution in original post

Highlighted
New Contributor

Thank you Robert, recursive call resolved my problem.

Reply
0 Kudos