Ungrouping/Flatenning GeoEvent Definitions
1557
3
01-15-2020 06:48 AM
You may work with GeoEvent Definitions that contain Groups. For example, here is a GeoEvent Definition called Duluth Vehicle Positions used for an AVL feed:
There is a Group called position that contains many fields within it. You may find the need to create an output feature service in a ArcGIS Online or a relational/spatiotemporal data store using GeoEvent. However, you will be unable to do so using a GeoEvent Definition that contains a group. Instead of going through the tedious task of recreating this GeoEvent Definition, you can use the below Python script to ungroup/flatten the GeoEvent Definition, which will essentially remove the Group types, but retain the underlying fields.
import arcpy, uuid, json, requests, xmltodict, re, os
# Disable warnings
requests.packages.urllib3.disable_warnings()
# Variables
xmlFile = os.path.join(arcpy.env.scratchFolder, "GeoEventDefinition.xml") # Temporary XML file that will be deleted
existingGeoEventDef = 'Duluth Vehicle Positions' # Existing GeoEvent Definition that contains groupings
newGeoEventDefName = 'Duluth Vehicle Positions Flat' # New GeoEvent Definition name
version = "11.1.0" # GeoEvent version
geoeventUsername = "geoeventAdmin" # Admin for GeoEvent Server (only applicable if GeoEvent is NOT federated)
geoeventPassword = "***********" # Admin Password (only applicable if GeoEvent is NOT federated)
geoeventServer = "geoevent.esri.com" # GeoEvent Server Fully Qualified Domain Name
portalUsername = "portaladmin" # Admin for Portal (only applicable if GeoEvent is federated)
portalPassword = "************" # Admin Password (only applicable if GeoEvent is federated)
portalServer = "portal.esri.com" # Portal Server Fully Qualified Domain Name (only applicable if GeoEvent is federated)
federated = True # Set to True if GeoEvent is federated with Portal, False otherwise
# Function to replace special characters with an _
def removeSpecialCharacters(name):
return re.sub(r'[^0-9a-zA-Z]+', r'_', name)
# Function to iterate through existing GeoEvent Definition and populate dictionary
def groupXML(definition, prefix):
if type(definition['fieldDefinitions']['fieldDefinition']) == list:
for defs in definition['fieldDefinitions']['fieldDefinition']:
if defs['@type'] == 'Group':
name = removeSpecialCharacters(defs['@name'])
prefix = f"{prefix}_{name}"
groupXML(defs, prefix)
else:
name = removeSpecialCharacters(defs['@name'])
# Update prefix if field is outside Group
for val in definition['fieldDefinitions']['fieldDefinition']:
if val['@name'] == prefix.split("_")[-1]:
if defs['@name'] not in val:
prefix = prefix.split("_")[-2]
fieldName = prefix + "_" + defs['@name']
fieldType = defs['@type']
cardinality = defs['@cardinality']
print(f"{prefix}_{name}")
geoeventDefDict.setdefault(fieldName, [])
geoeventDefDict[fieldName].append(fieldType)
geoeventDefDict[fieldName].append(cardinality)
elif type(definition['fieldDefinitions']['fieldDefinition']) == dict:
values = [vals for vals in definition['fieldDefinitions']['fieldDefinition'].values()]
if len(values) == 4 and type(values[3]) != dict:
fieldName = prefix + "_" + values[0]
fieldType = values[1]
cardinality = values[2]
print(prefix + "_" + values[0])
geoeventDefDict.setdefault(fieldName, [])
geoeventDefDict[fieldName].append(fieldType)
geoeventDefDict[fieldName].append(cardinality)
for val in values:
if type(val) == dict:
if type(val['fieldDefinition']) != list:
prefix = definition['@name'] + "_" + definition['fieldDefinitions']['fieldDefinition']['@name']
defs = definition['fieldDefinitions']['fieldDefinition']['fieldDefinitions']['fieldDefinition']
fieldName = f"{prefix}_{defs['@name']}"
fieldType = defs['@type']
cardinality = defs['@cardinality']
print(f"{prefix}_{defs['@name']}")
geoeventDefDict.setdefault(fieldName, [])
geoeventDefDict[fieldName].append(fieldType)
geoeventDefDict[fieldName].append(cardinality)
elif type(val['fieldDefinition']) == list:
for val2 in val['fieldDefinition']:
prefix = definition['@name'] + "_" + definition['fieldDefinitions']['fieldDefinition']['@name']
fieldName = f"{prefix}_{val2['@name']}"
fieldType = val2['@type']
cardinality = val2['@cardinality']
print(f"{prefix}_{val2['@name']}")
geoeventDefDict.setdefault(fieldName, [])
geoeventDefDict[fieldName].append(fieldType)
geoeventDefDict[fieldName].append(cardinality)
if __name__ == '__main__':
# Generate Token
if federated == True:
tokenURL = f'https://{portalServer}:7443/arcgis/sharing/rest/generateToken/'
params = {'f': 'pjson', 'username': portalUsername, 'password': portalPassword,
'referer': f'https://{geoeventServer}:6143/geoevent/admin'}
r = requests.post(tokenURL, data=params, verify=False)
response = json.loads(r.content)
token = response['token']
if federated == False:
tokenURL = f'https://{geoeventServer}:6443/arcgis/tokens/'
params = {'f': 'pjson', 'username': geoeventUsername, 'password': geoeventPassword,
'referer': f'https://{geoeventServer}:6143/geoevent/admin'}
r = requests.post(tokenURL, data=params, verify=False)
response = json.loads(r.content)
token = response['token']
# Header URL
postHeader = {'Content-Type': 'application/json', 'Cookie': 'adminToken=' + token}
# Get Original GeoEvent Definition GUID
getURL = f'https://{geoeventServer}:6143/geoevent/admin/geoeventdefinitions/{existingGeoEventDef}'
rp = requests.get(getURL, headers=postHeader, verify=False)
val = json.loads(rp.content)
origGuid = str(val[0]['guid'])
# Write GeoEventDefinition to XML file
getURL = f'https://{geoeventServer}:6143/geoevent/admin/geoeventdefinition/{origGuid}/.xml'
rp = requests.get(getURL, headers=postHeader, verify=False)
with open(xmlFile, 'wb') as file:
file.write(rp.content)
file.close()
# Generate new GUID
newGuid = str(uuid.uuid4())
# Create new GeoEventDefinition JSON
geoeventDefJSON = '{"guid":"' + newGuid + '","name": "' + newGeoEventDefName + \
'","owner": "auto-generated/com.esri.ges.adapter.inbound.Text/' \
+ version + '","accessType": "editable","fieldDefinitions": ['
# Create dictionary to store field names, types, and cardinalities
geoeventDefDict = {}
# Open xml file and convert to python dictionary
with open(xmlFile) as xml_file:
data_dict = xmltodict.parse(xml_file.read())
xml_file.close()
# Iterate through XML converted dictionary
for definition in data_dict['geoEventDefinition']['fieldDefinitions']['fieldDefinition']:
if definition['@type'] == 'Group':
prefix = removeSpecialCharacters(definition['@name'])
groupXML(definition, prefix)
elif definition['@type'] != 'Group':
fieldName = definition['@name']
fieldType = definition['@type']
cardinality = definition['@cardinality']
print(definition['@name'])
geoeventDefDict.setdefault(fieldName, [])
geoeventDefDict[fieldName].append(fieldType)
geoeventDefDict[fieldName].append(cardinality)
# Create final GeoEvent Definition JSON by iterating through geoeventDefDict dictionary
for key in geoeventDefDict.keys():
geoeventDefJSON += '{"name": "' + key + '","type": "' + geoeventDefDict[key][0] + '","cardinality": "' + \
geoeventDefDict[key][1] + '","fieldDefinitions": [{}],"fieldDefinitionTag": [""]},'
# Append trailing syntaxt to create valid JSON
geoeventDefJSON = geoeventDefJSON[0:-2] + '}]}'
# Send POST Request to create GeoEvent Definition
postURL = f'https://{geoeventServer}:6143/geoevent/admin/geoeventdefinition'
rp = requests.post(postURL, data=geoeventDefJSON, headers=postHeader, verify=False)
if rp.status_code == 200:
print(f"Successfully created {newGeoEventDefName} GeoEvent Definition")
# Delete xml file
os.remove(xmlFile)
After executing the above script, a new GeoEvent Definition called Duluth Vehicle Positions-Flat is created:
The Group position has been removed, but the underlying fields (bearing, latitude, longitude, etc) remain. This GeoEvent Definition can now be used to create feature services within the relational or spatiotemporal data stores.
Update 8/23/2023: The updated code will work with nested groups. Another module, xmltodict, is required to execute the script. If you are executing this script where you have ArcGIS Pro installed, you will need to install this module. First, you will need to clone the default environment. Once cloned, switch to this environment, search for the xmltodict module, and install:
Jake this is great! Can this be used on nested groups?
