Update Definition on Layer in Feature Service

[JohnMDye]

Have been tinkering with this for the better part of a day and can't figure it out. 

I'm calling an API to get a json response containing store locations, which contain the coordinates, then pushing that into a Feature Layer. Esri's implementation of the Spatially Enabled Dataframe's .to_featurelayer has the unfortunate behavior of sanitizing column names, even if the column names are valid to begin with.

The result is that its changing my column names from their original case to snake_case. For example 'storeName' gets changed to 'store_name' when I utilize the .to_featurelayer method to publish the data frame as a feature layer, even though there is absolutely nothing invalid at all about a feature layer with a field named 'storeName'.

Unlike the .to_featureclass method, where sanitize_columns is exposed as a parameter in the method and defaulted to True, meaning you can set it to False to avoid this behavior, sanitize_columns is defaulted to to True in the .to_featurelayer method and is not exposed as a parameter so there is no way to avoid it.

As a result, I'm trying to go back and update the column names using the update_definition method on the feature layer manager but I keep getting one of two errors. 

The first approach I took:

from arcgis.features import FeatureLayer
featureLayer =gis.content.get("c952e9e257bd4fc887be2934291548cb")
lyr = featureLayer.layers[0]
lyr.properties
originalDefinition = lyr.properties
# Get the fields array
originalFields = originalDefinition["fields"]
newFields = originalFields.copy()
for f in newFields:
    if "_" in f["name"]:
        newFieldName = f["name"].split("_")[0].lower()+f["name"].split("_")[1].title()
        f["name"] = newFieldName
#newFields
print('"fields": ' + json.dumps(newFields))
lyr.manager.update_definition('"fields": ' + json.dumps(newFields))

Fails with:

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-28-34cef8ca738c> in <module>
     13 #newFields
     14 print('"fields": ' + json.dumps(newFields))
---> 15 lyr.manager.update_definition('"fields": ' + json.dumps(newFields))

/opt/conda/lib/python3.7/site-packages/arcgis/features/managers.py in update_definition(self, json_dict)
   2002         u_url = self._url + "/updateDefinition"
   2003 
-> 2004         res = self._con.post(u_url, params)
   2005         self.refresh()
   2006         return res

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    718                                      file_name=file_name,
    719                                      try_json=try_json,
--> 720                                      force_bytes=kwargs.pop('force_bytes', False))
    721     #----------------------------------------------------------------------
    722     def put(self, url, params=None, files=None, **kwargs):

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in _handle_response(self, resp, file_name, out_path, try_json, force_bytes)
    512                     return data
    513                 errorcode = data['error']['code'] if 'code' in data['error'] else 0
--> 514                 self._handle_json_error(data['error'], errorcode)
    515             return data
    516         else:

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in _handle_json_error(self, error, errorcode)
    534 
    535         errormessage = errormessage + "\n(Error Code: " + str(errorcode) +")"
--> 536         raise Exception(errormessage)
    537     #----------------------------------------------------------------------
    538     def post(self,

Exception: Unable to update feature service layer definition.
Object reference not set to an instance of an object.
(Error Code: 400)

Kinda makes sense. Maybe I need to instantiate a FeatureLayer object on the actual layer...Let's try that:

featureLayer =gis.content.get("c952e9e257bd4fc887be2934291548cb")
lyr = FeatureLayer(featureLayer.layers[0])
originalDefinition = lyr.properties
# Get the fields array
originalFields = originalDefinition["fields"]
newFields = originalFields.copy()
for f in newFields:
    if "_" in f["name"]:
        newFieldName = f["name"].split("_")[0].lower()+f["name"].split("_")[1].title()
        f["name"] = newFieldName
newFields
print('"fields": ' + json.dumps(newFields))
#lyr.manager.update_definition('"fields": ' + json.dumps(newFields))

But it fails with:

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/arcgis/gis/__init__.py in _hydrate(self)
  11481                     if isinstance(self._con, Connection):
> 11482                         self._lazy_token = self._con.generate_portal_server_token(serverUrl=self._url)
  11483                     else:

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in generate_portal_server_token(self, serverUrl, expiration)
   1313             resp = self.post(path=self._token_url, postdata=postdata,
-> 1314                              ssl=True, add_token=False)
   1315         if isinstance(resp, dict) and resp:

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    719                                      try_json=try_json,
--> 720                                      force_bytes=kwargs.pop('force_bytes', False))
    721     #----------------------------------------------------------------------

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in _handle_response(self, resp, file_name, out_path, try_json, force_bytes)
    513                 errorcode = data['error']['code'] if 'code' in data['error'] else 0
--> 514                 self._handle_json_error(data['error'], errorcode)
    515             return data

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in _handle_json_error(self, error, errorcode)
    535         errormessage = errormessage + "\n(Error Code: " + str(errorcode) +")"
--> 536         raise Exception(errormessage)
    537     #----------------------------------------------------------------------

Exception: Unable to generate token.
'username' must be specified.
'password' must be specified.
'referer' must be specified.
(Error Code: 400)

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
<ipython-input-27-77bf06c45026> in <module>
      2 featureLayer =gis.content.get("c952e9e257bd4fc887be2934291548cb")
      3 lyr = FeatureLayer(featureLayer.layers[0], gis=gis)
----> 4 lyr.properties
      5 originalDefinition = lyr.properties
      6 # Get the fields array

/opt/conda/lib/python3.7/site-packages/arcgis/gis/__init__.py in properties(self)
  11460             return self._lazy_properties
  11461         else:
> 11462             self._hydrate()
  11463             return self._lazy_properties
  11464 

/opt/conda/lib/python3.7/site-packages/arcgis/gis/__init__.py in _hydrate(self)
  11507                     # try as a public server
  11508                     self._lazy_token = None
> 11509                     self._refresh()
  11510 
  11511                 except HTTPError as httperror:

/opt/conda/lib/python3.7/site-packages/arcgis/gis/__init__.py in _refresh(self)
  11450                     dictdata = self._con.get(self.url, params)
  11451                 else:
> 11452                     raise e
  11453 
  11454         self._lazy_properties = PropertyMap(dictdata)

/opt/conda/lib/python3.7/site-packages/arcgis/gis/__init__.py in _refresh(self)
  11443         else:
  11444             try:
> 11445                 dictdata = self._con.post(self.url, params, token=self._lazy_token)
  11446             except Exception as e:
  11447                 if hasattr(e, 'msg') and e.msg == "Method Not Allowed":

/opt/conda/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    619         try_json = kwargs.pop("try_json", True)
    620         add_token = kwargs.pop('add_token', True)
--> 621         if url.find('://') == -1:
    622             url = self._baseurl + url
    623         if kwargs.pop("ssl", False) or self._all_ssl:

AttributeError: 'FeatureLayer' object has no attribute 'find'

I don't understand why I would need to authenticate with credentials manually here or even how I would. I have a connection to a GIS object and even passing that in as the gis parameter on the instantiation of the FeatureLayer object at line 3 doesn't change result.

I also tried using FeatureCollection but got the same result in both approaches.

Help or insight appreciated.

[AndrewChapkowski · Accepted solution]

You cannot update the field names directly, you need to use a combination of update definition, add to definition and delete from definition to rename columns.

Let's start with the update call. 

update_def = {
    "fields": [
        {
            "name": "THISisWrong",
            "type": "esriFieldTypeString",
            "alias": "AmazingAlias",
            "length": 256,
            "editable": True,
            "nullable": True,
            "defaultValue": None,
            "description": None,
            "domain": None,
        }
    ]
}
res = flyer.manager.update_definition(update_def)
print(res)

This will update the alias column, though it doesn't directly answer your question.

Now how can we take say column: ThisIsWrong and Rename it to this_is_wrong? or something like that.  You'll have to do it in a few steps: 

1. Add the new field
2. Copy the column data
   1. If field is uneditable, make it uneditable
3. drop the old field

import re
from arcgis.gis import GIS
from arcgis.features import FeatureLayer

def _camelCase_to_underscore(name):
    """PEP8ify name"""
    if name[0].isdigit():
        name = "execute_" + name
    name = name.replace(" ", "_")
    if '_' in name:
        return name.lower()
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

if __name__ == "__main__":
    
    gis = GIS(profile='your_online_profile', verify_cert=False, trust_env=True)
    item = gis.content.get("e0d177f6b2ed4b1db1abe848dfea23f8")
    flyer = item.layers[0]
    
    #
    # Creating Columns with New Names
    #
    unchange_fields = [
        v
        for k, v in dict(flyer.properties).items()
        if k in ['objectIdField', 'globalIdField']
    ]
    rename_fields = [
        dict(fld)
        for fld in flyer.properties.fields
        if fld["type"] not in ('esriFieldTypeOID', 'esriFieldTypeGlobalID')
        or fld['name'] not in (unchange_fields)
    ]
    expressions = []
    delete_exp = {"fields": []}
    add_fields = []
    uneditable = []
    for idx, fld in enumerate(rename_fields):
        if _camelCase_to_underscore(fld["name"]) != fld["name"].lower():
    
            expressions.append(
                {
                    "field": _camelCase_to_underscore(fld["name"]),
                    "sqlExpression": fld["name"],
                }
            )
            delete_exp["fields"].append({"name": fld["name"]})
    
            fld['name'] = _camelCase_to_underscore(fld["name"])
            fld['alias'] = _camelCase_to_underscore(fld["alias"])
            if fld['editable'] == False:
                uneditable.append(fld['name'])
            fld['editable'] = True
            add_fields.append(fld)
        else:
            print(f"Skipping field because it meets the format standards: {fld['name']}")
    
    print("Add the renamed columns")
    
    res = flyer.manager.add_to_definition({"fields": add_fields})
    print(res)
    print("Update the column values")
    job = flyer.calculate(where="1=1", calc_expression=expressions, future=True)
    result = job.result()
    print(result)
 
    # Set fields that are not editable back to original state.
    #
    update_editable = [fld for fld in add_fields if fld['name'] in uneditable]
    if update_editable:
        for fld in update_editable:
            fld['editable'] = True
        update_def = {"fields": update_editable}
    
        res = flyer.manager.update_definition(update_def)
        print(res)
    #  Drop the Old Fields
    #
    print(flyer.manager.delete_from_definition(delete_exp))
    flyer._refresh()
    print([fld['name'] for fld in flyer.properties.fields])

It should be noted that this code doesn't handle cases.  If you have a field called Editor the caml casing should be editor.  The underlying database sees the field as the same name and thus can't be renamed.