topic Multiple queryTask operations with maxRecordCount in ArcGIS JavaScript Maps SDK Questions

Multiple queryTask operations with maxRecordCount

BlakeTerhune — Mon, 01 Oct 2018 18:46:39 GMT

I tried searching but was surprised to not find any good results. Does anyone have a coding pattern to handle "paging" through a large query that exceeds the service's maxRecordCount property? I'd like to find a way to query the record count first, then build a bunch of query tasks with varying start and num properties to retrieve all the results. Promise.all() seems like it'd be what's needed but I can't quite wrap my head around how to make it all work. Here's my related question regarding distinct values that started this.

Re: Multiple queryTask operations with maxRecordCount

RobertScheitlin__GISP — Sun, 12 Dec 2021 06:41:18 GMT

Blake,

Here is how I do that:

var iMaxRecords = 0;
var featuresProcessed = 0;
var featuresTotal = 0;
var maxRecordCount = layer.maxRecordCount; //normally 1000

queryTask = new QueryTask(uri);
var cntQuery = new Query();
cntQuery.where = "1=1";
queryTask.executeForCount(cntQuery, function(count){
  featuresTotal = count;
  if(count <= maxRecordCount){
    //not an issue where you need to page
  }else{
    query.returnGeometry = false;
    query.objectIds = null;
    query.where = "1=1";
    queryTask.executeForIds(query, onSearchIdsFinish, onSearchError);
  }
});

function onSearchIdsFinish(results){
  if (results.length > 0) {
    objectIdsArray = results;
    featuresTotal = objectIdsArray.length;
    query.where = null;
    query.text = null;
    query.objectIds = objectIdsArray.slice(0, maxRecordCount);
    queryTask.execute(query, onSearchFinish, onSearchError);
  } else {
    console.error('onSearchIdsFinish returned zero length');
    isQuerying = false;
    esc = false;
  }
}‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

function onSearchFinish(featureSet) {
// do something with the results
  featuresProcessed += featureSet.features.length;

  if (iMaxRecords === 0) {
    iMaxRecords = featuresProcessed;
    iStart += iMaxRecords;
  }
  if (iStart < objectIdsArray.length) {
            //If we get this far we need to requery the server for the next lot of records
    query.objectIds = objectIdsArray.slice(iStart, iStart + iMaxRecords);
    queryTask.execute(query, onSearchFinish, onSearchError);

    iStart += iMaxRecords;
  }
}‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

Re: Multiple queryTask operations with maxRecordCount

BlakeTerhune — Sun, 12 Dec 2021 06:41:21 GMT

Interesting approach, thanks. My feature layer (hosted at 10.5.1) doesn't have a maxRecordCount property. It doesn't even have a capabilities property. Any thoughts on that?

After posting I got something that I think works. Besides the fact it doesn't work with returnDistinctValues (as I mentioned before), I'm not sure how I feel about it.

var maxRecordCount = 1000;
var queryTask = new QueryTask();
queryTask.url = 'http://server/arcgis/rest/services/ServiceName/MapServer/0';
var queryParams = new Query({
  returnGeometry: false,
  outFields: ['someFieldName'],
  where: 'someFieldName is not null',
});
queryTask.executeForCount(queryParams)
.then(function(queryRecordCount) {
  var pageCount = Math.ceil(queryRecordCount / maxRecordCount);
  var resultPages = [];
  for (var i = 0; i < pageCount; i++) {
    resultPages.push(i * maxRecordCount);
  }
  return Promise.all(resultPages.map(function(resultPageStart){
    queryParams.start = resultPageStart;
    queryParams.num = maxRecordCount - 1;
    return queryTask.execute(queryParams);
  }))
})
.then(function(featureSets) {
  // collect features from each FeatureSet
  var features = [];
  featureSets.forEach(function(featureSet) {
    features = features.concat(featureSet.features);
  })
  // do something with array of features
})
.catch(function(err) {
  console.error(err);
});‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

Re: Multiple queryTask operations with maxRecordCount

RobertScheitlin__GISP — Mon, 01 Oct 2018 23:26:59 GMT

Blake,

You can use QueryTask.exectueForCount() and in the Query object you can use returnDistinct. This will allow you to get a number that is in fact the real distinct count for your query. You then can decide if you need to use the paging route. Also the query result returns a exceededTransferLimit property that tells you if your results are not all the distinct data.