Arcade data expression not returning all records to feature set.

MS_PTOI · ‎02-09-2023

I'm working on a dashboard that records spawning data of salmon. As part of the dashboard I need to be able to call in survey records from a table, determine if the survey year is even or odd and then apply a transformation specific to a species and differs by odd and even years.

The following code seemingly functions as expected in that it applies the appropriate transformation.

However, not all records are being returned by the data expression. I can't figure out why some records aren't being returned and its consistently the same records which are excluded (several dozen of the 1900+ records in the original table). Any insights into why this is happening or how to optimize the code would be greatly appreciated.

var p = Portal('https://www.arcgis.com/');
var fs = FeatureSetByPortalItem(
    p,
    'XYZ',
    0,
    ['StreamID_fk',
    'DateSurveyed',
    'Species',
    'TotalRedds'
    ],
    false
    );
    
var features = [];
var feat;

for (var r in fs) {
    var d = r['DateSurveyed']
    var sd = Number(d)
    var oey = Number(Year(d))
    var stream = r['StreamID_fk']
    var fishType = r['Species']
    var reddNum = r['TotalRedds']
    
    function isOdd(value) {
        if (Boolean(value % 2)) {
            if (fishType == 'Steelhead') {
                return reddNum * 1.62
            } else {
                return reddNum * 1
            }
        } else {
           if (fishType == 'Chinook') {
                return reddNum * 2.5
            } else if (fishType == 'Steelhead'){
                return reddNum * 1.62
            } else {
                return reddNum * 1
            }
        }
    }
    
    var esc = isOdd(oey)
    
    feat = {
        attributes: {
            'StreamID': stream,
            'SurveyDate': sd,
            'FishSpecies': fishType,
            'ReddNums': reddnum,
            'EscValue': esc,
        }
    }
    Push(features, feat)
}

var Dict = {
  'fields':[
        {'name': 'StreamID', 'type': 'esriFieldTypeString'},
        {'name': 'SurveyDate', 'type': 'esriFieldTypeDate'},
        {'name': 'FishSpecies', 'type': 'esriFieldTypeString'},
        {'name': 'ReddNums', 'type': 'esriFieldTypeInteger'},
        {'name': 'EscValue', 'type': 'esriFieldTypeInteger'},],
        'geometryType': '',
        'features': features,
};

return FeatureSet(Text(Dict));

EvelynHsu · ‎02-09-2023

I see that your expression is trying to perform data transformation based on certain field values and then map them into a slightly different feature set schema.

Have you tried looking into data records that were NOT returned by the expression, and see if they share certain common characteristics (e.g. in terms of field values upper vs lower case etc.)? I would guess the problem might lie with either the data you are using OR the expression logics (i.e. it doesn't perform according to how you'd like it to).

TanGnar · ‎02-24-2023

I ran into this issue and found that I was expecting integers, but those entering data were sometimes entering decimals. Those records were not included in FeatureSet because they did not match the esri field type I had specified.

After a quick glance, It looks like you might be using decimals in the reddNums field, which you have set to esriFieldTypeInteger. If any values get passed to it that aren't integers, it will exclude those records from the feature set. You may need to change any fields expecting decimal numbers to field type Double of Float.

lannguyentl · ‎06-10-2024

Hi All,

I have the same issue when merging data from different services together in Enterprise Dashboard. The final FeatureSet shows wrong numbers of rows for some feature services. However, when I tested with one by one feature service, using the same code, it worked well. Is there any restriction for this combined FeatureSet? It is useful but not reliable.

Thanks,

Lan

// Write an expression that returns a FeatureSet.
// Documentation: https://arcg.is/3c419TD
// Samples: https://arcg.is/38SEWWz

// Portal
var p = Portal('https://abc.com/portal');

// Create FeatureSet for points 
//HarvestManagers 01 Site Visit
var fs1 = FeatureSetByPortalItem(p,'179e25877e02a9a', 0, ['globalid','objectid','created_user','created_date'], false);

//HarvestManagers 02 HS Quarterly Audit
var fs2 = FeatureSetByPortalItem(p,'e88d15ca561fa1761', 0, ['globalid','objectid','created_user','created_date'], false);

//HarvestManagers 03 Log Quality Audit
var fs3 = FeatureSetByPortalItem(p,'0d721aaed57af9', 0, ['globalid','objectid','created_user','created_date'], false);

//HarvestManagers 04 Stem Quality Audit 
var fs4 = FeatureSetByPortalItem(p,'290836073a3c4', 0, ['globalid','objectid','created_user','created_date'], false);

//HarvestManagers 05 Stock Accuracy Audit 
var fs5 = FeatureSetByPortalItem(p,'3114fde29a827', 0, ['globalid','objectid','created_user','created_date'], false);

//HarvestManagers 07 Cut Plan Audit
var fs6 = FeatureSetByPortalItem(p,'8ef9f85610324a5c9c7fd3256fa38413', 0, ['globalid','objectid','created_user','created_date'], false);

//Production 01 Environmental Audit
var fs7 = FeatureSetByPortalItem(p,'73977d889d06d50fab', 0, ['globalid','objectid','created_user','created_date'], false);


// Create empty feature array and feat object for output
var features = [];
var feat;

// Iterate fs1
for (var feature in fs1) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 01 Site Visit': "HarvestManagers 01 Site Visit",
            'survey_name':"HarvestManagers 01 Site Visit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};

// Iterate fs2
for (var feature in fs2) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 02 HS Quarterly Audit': "HarvestManagers 02 HS Quarterly Audit",
            'survey_name':"HarvestManagers 02 HS Quarterly Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};

// Iterate fs3
for (var feature in fs3) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 03 Log Quality Audit': "HarvestManagers 03 Log Quality Audit",
            'survey_name':"HarvestManagers 03 Log Quality Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};

// Iterate fs4
for (var feature in fs4) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 04 Stem Quality Audit': "HarvestManagers 04 Stem Quality Audit",
            'survey_name':"HarvestManagers 04 Stem Quality Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};


// Iterate fs5
for (var feature in fs5) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 05 Stock Accuracy Audit': "HarvestManagers 05 Stock Accuracy Audit",
            'survey_name':"HarvestManagers 05 Stock Accuracy Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};

// Iterate fs6
for (var feature in fs6) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'HarvestManagers 07 Cut Plan Audit': "HarvestManagers 07 Cut Plan Audit",
            'survey_name':"HarvestManagers 07 Cut Plan Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};

// Iterate fs7
for (var feature in fs7) {
    
    // Create feature with aggregated values
    feat = { 
        'attributes': { 
            
            'objectid': feature['objectid'],
            'globalid': feature['globalid'],
            'Production 01 Environmental Audit': "Production 01 Environmental Audit",
            'survey_name':"Production 01 Environmental Audit",
            'created_user': feature['created_user'],
            'created_date': NUMBER(feature['created_date']),
            
        }
    };
    
    // Push feature into array
    Push(features, feat);
};


// Create dict for output FeatureSet
var out_dict = { 
    'fields': [
        
        {'name': 'objectid', 'type': 'esriFieldTypeOID'},
        {'name': 'globalid', 'type': 'esriFieldTypeGlobalID'},
        {'name': 'HarvestManagers 01 Site Visit', 'type': 'esriFieldTypeString'},
        {'name': 'HarvestManagers 02 HS Quarterly Audit', 'type': 'esriFieldTypeString'},
        {'name': 'HarvestManagers 03 Log Quality Audit', 'type': 'esriFieldTypeString'},
        {'name': 'HarvestManagers 04 Stem Quality Audit', 'type': 'esriFieldTypeString'},
        {'name': 'HarvestManagers 05 Stock Accuracy Audit', 'type': 'esriFieldTypeString'},
        {'name': 'HarvestManagers 07 Cut Plan Audit', 'type': 'esriFieldTypeString'},
        {'name': 'Production 01 Environmental Audit', 'type': 'esriFieldTypeString'},
        {'name': 'survey_name', 'type': 'esriFieldTypeString'},
        {'name': 'created_user', 'type': 'esriFieldTypeString'},
        {'name': 'created_date', 'type': 'esriFieldTypeDate'},
        
    ],
  'geometryType': '', 
  'features': features 
}; 

// Convert dictionary to feature set. 
return FeatureSet(Text(out_dict));