How to quickly query features from feature service?

VincentLantaca1 · ‎05-28-2019

Hello. I am trying to figure out a quick way to query features that are on a feature service or map service so that I can go on to create visualizations from them.

Feature services have a limit of how many records you can query at once. This limit is usually 1000. So, as a result I have to keep re-querying until I have all records. So far one way I found to do this is using QueryTask. However it seems quite slow compared to operations dashboard.

Is this a bad approach?

Here is my current method for querying records:

<html>
<head>
  <meta name="description" content="DevLav: Query a feature layer">
  <meta charset="utf-8">
  <meta name="viewport" content="initial-scale=1,maximum-scale=1,user-scalable=no">
  <title>Query All using QueryTask</title>
  <style>
    html,
    body,
    #viewDiv {
      padding: 0;
      margin: 0;
      height: 100%;
      width: 100%;
    }
  </style>
  <link rel="stylesheet" href="https://js.arcgis.com/4.11/esri/css/main.css">
  <script src="https://js.arcgis.com/4.11/"></script>
</head>

<script>
  require([
    "esri/Map",
    "esri/views/MapView",
    "esri/tasks/support/Query",
    "esri/tasks/QueryTask",
    "esri/Graphic",
    "dojo/domReady!"
  ],
  function(
    Map, 
    MapView,
    Query,
    QueryTask,
    Graphic
  ){
    serviceUrl = "https://services.arcgis.com/V6ZHFr6zdgNZuVG0/ArcGIS/rest/services/earthquakes_geojson/FeatureServer/0";

    var map = new Map({
      basemap: "topo-vector"
    });

    var view = new MapView({
      container: "viewDiv",
      map: map
    });

    // gets a time delta dictionary between two Date objects, 'start' and 'stop'
    function getTimeDelta(start, stop){
      var timeDelta = parseInt(stop - start);
      var days = parseInt(timeDelta / 86400000);
      var daysRem = timeDelta - (days * 86400000);
      var hours = parseInt(daysRem / 3600000);
      var hoursRem = daysRem - (hours * 3600000);
      var minutes = parseInt(hoursRem / 60000);
      var minutesRem = hoursRem - (minutes * 60000);
      var seconds = parseInt(minutesRem / 1000);
      var milliseconds = minutesRem - (seconds * 1000);

      var results = {
        days: days,
        hours: hours,
        minutes: minutes,
        seconds: seconds,
        milliseconds: milliseconds
      }

      return results;
    }

    // return Promise<Number> (should be total number of records, since OBJECTID should never be blank)
    function getNumRecords(){
      var query = new Query();
      query.where = "'OBJECTID' LIKE '%'";
      query.outFields = ['*'];
      query.returnGeometry = true;
      var queryTask = new QueryTask({
        url: serviceUrl
      });

      return queryTask.executeForCount(query);
    }

    function createQueryFunction(startId, numRecords){
      return function(){
        var query = new Query();
        query.start = startId;
        query.num = numRecords;
        query.outFields = ['*'];
        query.returnGeometry = true;
        var queryTask = new QueryTask({
          url: serviceUrl
        });

        return queryTask.execute(query)
      }
    }

    async function doWork(promises){
      var results = [];

      for (var promise of promises){
        result = await promise();
        results = results.concat(result.features);
      }

      return results;
    }

    var startTime = new Date();

    getNumRecords().then(function(numRecords){
      console.log(numRecords);

      var promises = [];
      for (var i=0; i<numRecords; i += 1000){
        promises.push(createQueryFunction(i, 1000));
      }

      doWork(promises).then(results => {
        console.log('results', results);
        var stopTime = new Date();
        console.log(getTimeDelta(startTime, stopTime));
      });
    });
  });
</script>
</head>

<body>
  <div id="viewDiv"></div>
</body>

</html>‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

JackFairfield · ‎05-29-2019

Hey Vincent,

Have you looked into ESRI's Rest JS library? It is useful when you don't necessarily need to show a map.

You could do something like this to fetch all of the features quickly:

https://jsbin.com/sehunejone/edit?html,output

<!DOCTYPE html>
<html>

<head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width">
    <title>ArcGIS REST JS - AMD</title>
</head>

<body>
    Open your console to see the demo.
</body>
<script src="https://unpkg.com/@esri/arcgis-rest-request@2.0.3/dist/umd/request.umd.js"></script>
<script src="https://unpkg.com/@esri/arcgis-rest-feature-layer@2.0.3/dist/umd/feature-layer.umd.js"></script>
<script>
    var url = "https://services.arcgis.com/V6ZHFr6zdgNZuVG0/ArcGIS/rest/services/earthquakes_geojson/FeatureServer/0";

    async function GetRecordCount() {
        var { count } = await arcgisRest.queryFeatures({
            url: url,
            returnCountOnly: true
        });
        return count;
    }

    async function GetData() {

        let numRecords = await GetRecordCount();
        let promises = [];
        let data = [];
        const maxRecordCount = 1000;

        for (var i = 0; i < numRecords; i += maxRecordCount) {
            let q = arcgisRest.queryFeatures({
                url: url,
                resultOffset: i,
                resultRecordCount: maxRecordCount
            });
            promises.push(q);
        }

        let dataArr = await Promise.all(promises);
        for (const res of dataArr) {
            data = data.concat(res.features);
        }
        return data;
    }
    var t0 = performance.now();
    GetData().then(function(res) {
        var t1 = performance.now();
        console.log("Query took " + Math.floor(t1 - t0) + " milliseconds.");
        console.log(res);
    })
</script>

</html>

View solution in original post

JackFairfield · ‎05-29-2019

One thing you can do to slightly improve performance is to make sure all of your requests aren't waiting for the previous query to finish before they run. You can change the doWork function to something like this:

async function doWork(promiseFunctions) {
    var promises = [];
    for (var func of promiseFunctions) {
        promises.push(func());
    }
    var results = [];
    for (var result of await Promise.all(promises)) {
        results = results.concat(result.features);
    }
    return results;
}

This way, you are awaiting all of the promises at once instead of doing them in order and having to await each one.

VincentLantaca1 · ‎05-29-2019

Thanks. One reason I didn't do this was that I was worried about CPU usage. However this seems to work fine. Oddly enough, operations dashboards seem to create visualizations based on all records yet they load fast and don't use up as much CPU. I'm wondering if there are other approaches to querying records with ArcGIS JS.

JackFairfield · ‎05-29-2019

Hey Vincent,

Have you looked into ESRI's Rest JS library? It is useful when you don't necessarily need to show a map.

You could do something like this to fetch all of the features quickly:

https://jsbin.com/sehunejone/edit?html,output

<!DOCTYPE html>
<html>

<head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width">
    <title>ArcGIS REST JS - AMD</title>
</head>

<body>
    Open your console to see the demo.
</body>
<script src="https://unpkg.com/@esri/arcgis-rest-request@2.0.3/dist/umd/request.umd.js"></script>
<script src="https://unpkg.com/@esri/arcgis-rest-feature-layer@2.0.3/dist/umd/feature-layer.umd.js"></script>
<script>
    var url = "https://services.arcgis.com/V6ZHFr6zdgNZuVG0/ArcGIS/rest/services/earthquakes_geojson/FeatureServer/0";

    async function GetRecordCount() {
        var { count } = await arcgisRest.queryFeatures({
            url: url,
            returnCountOnly: true
        });
        return count;
    }

    async function GetData() {

        let numRecords = await GetRecordCount();
        let promises = [];
        let data = [];
        const maxRecordCount = 1000;

        for (var i = 0; i < numRecords; i += maxRecordCount) {
            let q = arcgisRest.queryFeatures({
                url: url,
                resultOffset: i,
                resultRecordCount: maxRecordCount
            });
            promises.push(q);
        }

        let dataArr = await Promise.all(promises);
        for (const res of dataArr) {
            data = data.concat(res.features);
        }
        return data;
    }
    var t0 = performance.now();
    GetData().then(function(res) {
        var t1 = performance.now();
        console.log("Query took " + Math.floor(t1 - t0) + " milliseconds.");
        console.log(res);
    })
</script>

</html>

VincentLantaca1 · ‎05-29-2019

This is great, I had no idea this existed!