Network Analyst in Python Loop becomes exponentially slower

FrankCardone1 · ‎09-25-2023

Hi Everyone,

I have an ArcGIS Python Toolbox that loops through a Location-Allocation problem, to analyze changes in allocation based on our growth forecasts. When I run it, the first iteration of the loop takes 10 minutes, but every subsequent loop takes much longer than the previous. By the 10th iteration, it takes multiple hours. Any idea on what is going wrong?

Code:

import arcpy

import os
import pandas as pd
import numpy as np
import arcgis
from arcgis.features import GeoAccessor, GeoSeriesAccessor



def location_allocation_tool(nd_layer, travel_mode, geodatabase, demand, scc_layer, line_barriers, search_cutoff, site_count=20):
    if 'yr0' in demand:
        year_tag = 'Capacity_Current'
    else:
        year_tag = demand.split("_")[-1]   
    print(year_tag)
    demand_lyr = 'demand_lyr'
    capacity_column = [x.name for x in arcpy.ListFields(scc_layer) if year_tag in x.name][0]
    arcpy.AddMessage("begin optimization for {}...".format(demand))
    arcpy.MakeFeatureLayer_management(demand, demand_lyr)
    out_cluster = "cluster_" + demand
    arcpy.AddJoin_management(demand_lyr, "OBJECTID", out_cluster, "SOURCE_ID")
    cluster_expression = [x.name for x in arcpy.ListFields(demand_lyr) if "CLUSTER" in x.name][0] + " = -1"
    arcpy.SelectLayerByAttribute_management(demand_lyr, "NEW_SELECTION", cluster_expression)
    mean_centers = "meancenter_" + demand
    mean_centers_lyr = mean_centers + "_lyr"
    arcpy.MakeFeatureLayer_management(mean_centers, mean_centers_lyr, arcpy.AddFieldDelimiters(mean_centers, "CLUSTER_ID") + " >= 0") # clustered stores

    loc_alloc = arcpy.nax.LocationAllocation(nd_layer)
    loc_alloc.accumulateAttributeNames = ['miles']
    loc_alloc.travelMode = travel_mode
    loc_alloc.travelDirection = arcpy.nax.TravelDirection.FromFacility
    loc_alloc.problemType = arcpy.nax.LocationAllocationProblemType.MaximizeCapacitatedCoverage
    loc_alloc.facilityCount = site_count
    loc_alloc.defaultImpedanceCutoff = search_cutoff
    loc_alloc.lineShapeType = arcpy.nax.LineShapeType.StraightLine
    loc_alloc.distanceUnits = arcpy.nax.DistanceUnits.Miles
    loc_alloc.timeUnits = arcpy.nax.TimeUnits.Minutes

    field_mappings_facilities = loc_alloc.fieldMappings(arcpy.nax.LocationAllocationInputDataType.Facilities)
    field_mappings_facilities["Name"].mappedFieldName = "scc_number"
    field_mappings_facilities["Capacity"].mappedFieldName = capacity_column
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.Facilities, scc_layer, field_mappings_facilities, append=False)

    #load non clustered stores
    field_mappings_demand = loc_alloc.fieldMappings(arcpy.nax.LocationAllocationInputDataType.DemandPoints, use_location_fields=True)
    field_mappings_demand["Name"].mappedFieldName = demand + ".store_number"
    field_mappings_demand["Weight"].mappedFieldName = demand + ".avg_weekly_net_weight_dough"
    for field in ['SourceID', 'SourceOID', 'PosAlong', 'SideOfEdge', 'CurbApproach', 'TimeCutoff', 'DistanceCutoff', 'Bearing', 'BearingTol', 'NavLatency', 'Cutoff']:
        field_mappings_demand[field].mappedFieldName = demand + ".{}".format(field)
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.DemandPoints, demand_lyr, field_mappings_demand, append=False)

    #load clustered stores
    loc_alloc.setLocateSettingsOverrides(arcpy.nax.LocationAllocationInputDataType.DemandPoints, allow_auto_relocate=True, search_tolerance=16000)
    field_mappings_clusters = loc_alloc.fieldMappings(arcpy.nax.LocationAllocationInputDataType.DemandPoints, use_location_fields=True)
    field_mappings_clusters["Name"].mappedFieldName = "CLUSTER_ID"
    field_mappings_clusters["Weight"].mappedFieldName = "SUM_" + demand + "_avg_weekly_net_weight_dough"
    for field in ['SourceID', 'SourceOID', 'PosAlong', 'SideOfEdge', 'CurbApproach', 'TimeCutoff', 'DistanceCutoff', 'Bearing', 'BearingTol', 'NavLatency', 'Cutoff']:
        field_mappings_clusters[field].mappedFieldName = demand + ".{}".format(field)
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.DemandPoints, mean_centers_lyr, field_mappings_clusters)
    
    #load_line_barriers
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.LineBarriers, line_barriers)

    result = loc_alloc.solve()     

    if result.solveSucceeded:
        arcpy.AddMessage("{} complete".format(demand))
        result.export(arcpy.nax.LocationAllocationOutputDataType.Lines, os.path.join(geodatabase, "line_results_{}".format(demand)))
        line_fields = result.fieldNames(arcpy.nax.LocationAllocationOutputDataType.Lines)
        line_data = [row for row in result.searchCursor(arcpy.nax.LocationAllocationOutputDataType.Lines, line_fields)]
        lines_df = pd.DataFrame(line_data, columns=line_fields)

        demand_df = pd.DataFrame.spatial.from_featureclass(demand)
        clusters_df = pd.DataFrame.spatial.from_featureclass(out_cluster)
        store_assignments_df = pd.merge(demand_df[['OBJECTID', 'store_number', 'avg_weekly_net_weight_dough', 'build_year']], 
                    clusters_df[['SOURCE_ID', 'CLUSTER_ID']], 
                    left_on='OBJECTID', 
                    right_on='SOURCE_ID', 
                    how='left')
        
        store_assignments_df['DEMAND_ID'] = np.where(store_assignments_df['CLUSTER_ID'] == -1, 
                                                        store_assignments_df['store_number'], 
                                                        store_assignments_df['CLUSTER_ID']).astype(int)

        lines_df[['ORIGIN', 'DESTINATION']] = lines_df["Name"].str.split(" - ", expand=True).astype(int)
        store_assignments_df = pd.merge(store_assignments_df[['store_number', 'avg_weekly_net_weight_dough', 'build_year',  'DEMAND_ID']],
                                        lines_df[['ORIGIN', 'DESTINATION']],
                                        how='left',
                                        left_on='DEMAND_ID',
                                        right_on='DESTINATION')


        #result_dict[demand]['lines'] = lines_df
        print(result.solverMessages(arcpy.nax.MessageSeverity.All))

       
    else:
        arcpy.AddMessage("Solve failed")
        arcpy.AddMessage(result.solverMessages(arcpy.nax.MessageSeverity.All))
        store_assignments_df =  pd.DataFrame(columns= ['store_number', 'avg_weekly_net_weight_dough', 'build_year',  'DEMAND_ID', 'ORIGIN', 'DESTINATION'])
        #unchosen_facilities = []
    arcpy.management.RemoveJoin(demand_lyr)
    arcpy.Delete_management(demand_lyr)
    arcpy.Delete_management(mean_centers_lyr)
    del result, loc_alloc, field_mappings_facilities, field_mappings_demand, field_mappings_clusters

    return store_assignments_df

def optimize_network(network_dataset, workspace, scc_locations, future_demand_list, line_barriers='line_barriers', search_cutoff_miles=800, site_count=20):
    arcpy.env.workspace = workspace
    nd_layer_name = "Routing_ND"
    scc_lyr = 'scc_lyr'
    line_lyr = "line_lyr"
    # demand_lyr = 'demand_lyr'
    arcpy.MakeFeatureLayer_management(scc_locations, scc_lyr)
    arcpy.MakeFeatureLayer_management(line_barriers, line_lyr)
    arcpy.nax.MakeNetworkDatasetLayer(network_dataset, nd_layer_name)
    nd_travel_modes = arcpy.nax.GetTravelModes(nd_layer_name)
    travel_mode = arcpy.nax.TravelMode(nd_travel_modes['Truck Distance'])
    params = travel_mode.attributeParameters
    params[('Driving a Truck', 'Restriction Usage')] = 5
    travel_mode.attributeParameters = params
    travel_mode.useHierarchy = True

    #future_demand_list = arcpy.ListFeatureClasses("Stores*", feature_type='Point', feature_dataset="Store_Data")
    arcpy.AddMessage("beginning model run for {} years...".format(len(future_demand_list)))

    result_dict = {}

    for demand in future_demand_list:
        result_dict[demand] = location_allocation_tool(nd_layer=nd_layer_name,
                                                       travel_mode=travel_mode,
                                                       geodatabase=workspace,
                                                       demand=demand,
                                                       scc_layer=scc_lyr,
                                                       line_barriers=line_lyr,
                                                       search_cutoff=search_cutoff_miles,
                                                       site_count=site_count)
        #result_dict[demand] = solve_result
    arcpy.Delete_management(nd_layer_name)
    arcpy.Delete_management(scc_lyr)
    arcpy.Delete_management(line_lyr)   
    return result_dict

And here is a simplified version if that's too much (just an example of my workflow):

## simplified code that doesnt run, just shows a pared down workflow
arcpy.env.workspace = workspace
nd_layer_name = "Routing_ND"
scc_lyr = 'scc_lyr'
line_lyr = "line_lyr"
# demand_lyr = 'demand_lyr'
arcpy.MakeFeatureLayer_management(scc_locations, scc_lyr)
arcpy.MakeFeatureLayer_management(line_barriers, line_lyr)
arcpy.nax.MakeNetworkDatasetLayer(network_dataset, nd_layer_name)
nd_travel_modes = arcpy.nax.GetTravelModes(nd_layer_name)
travel_mode = arcpy.nax.TravelMode(nd_travel_modes['Truck Distance'])
params = travel_mode.attributeParameters
params[('Driving a Truck', 'Restriction Usage')] = 5
travel_mode.attributeParameters = params
travel_mode.useHierarchy = True

result_dict = {}

# loop through multiple years of demand
for demand in future_demand_list:
    
    arcpy.AddMessage("begin optimization for {}...".format(demand))
    arcpy.MakeFeatureLayer_management(demand, demand_lyr)

    loc_alloc = arcpy.nax.LocationAllocation(nd_layer_name)
    loc_alloc.accumulateAttributeNames = ['miles']
    loc_alloc.travelMode = travel_mode
    loc_alloc.travelDirection = arcpy.nax.TravelDirection.FromFacility
    loc_alloc.problemType = arcpy.nax.LocationAllocationProblemType.MaximizeCapacitatedCoverage
    loc_alloc.facilityCount = 20
    loc_alloc.defaultImpedanceCutoff = 800
    loc_alloc.lineShapeType = arcpy.nax.LineShapeType.StraightLine
    loc_alloc.distanceUnits = arcpy.nax.DistanceUnits.Miles
    loc_alloc.timeUnits = arcpy.nax.TimeUnits.Minutes

    field_mappings_facilities = loc_alloc.fieldMappings(arcpy.nax.LocationAllocationInputDataType.Facilities)
    field_mappings_facilities["Name"].mappedFieldName = "scc_number"
    field_mappings_facilities["Capacity"].mappedFieldName = "capacity_column"
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.Facilities, scc_layer, field_mappings_facilities, append=False)

    field_mappings_demand = loc_alloc.fieldMappings(arcpy.nax.LocationAllocationInputDataType.DemandPoints, use_location_fields=True)
    field_mappings_demand["Name"].mappedFieldName = demand + ".store_number"
    field_mappings_demand["Weight"].mappedFieldName = demand + ".avg_weekly_net_weight_dough"
    for field in ['SourceID', 'SourceOID', 'PosAlong', 'SideOfEdge', 'CurbApproach', 'TimeCutoff', 'DistanceCutoff', 'Bearing', 'BearingTol', 'NavLatency', 'Cutoff']:
        field_mappings_demand[field].mappedFieldName = demand + ".{}".format(field)
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.DemandPoints, demand_lyr, field_mappings_demand, append=False)


    #load_line_barriers
    loc_alloc.load(arcpy.nax.LocationAllocationInputDataType.LineBarriers, line_barriers)

    result = loc_alloc.solve()     

    if result.solveSucceeded:
        arcpy.AddMessage("{} complete".format(demand))
        result.export(arcpy.nax.LocationAllocationOutputDataType.Lines, os.path.join(geodatabase, "line_results_{}".format(demand)))
        demand_df = pd.DataFrame.spatial.from_featureclass(demand)
        result_dict[demand] = demand_df
        print(result.solverMessages(arcpy.nax.MessageSeverity.All))

       
    else:
        arcpy.AddMessage("Solve failed")
        arcpy.AddMessage(result.solverMessages(arcpy.nax.MessageSeverity.All))
        store_assignments_df =  pd.DataFrame(columns= ['store_number', 'avg_weekly_net_weight_dough', 'build_year',  'DEMAND_ID', 'ORIGIN', 'DESTINATION'])
        #unchosen_facilities = []
    arcpy.management.RemoveJoin(demand_lyr)
    arcpy.Delete_management(demand_lyr)
    del result, loc_alloc, field_mappings_facilities, field_mappings_demand

FrankCardone1 · ‎09-26-2023

Ask away! whatever it takes to find out whats happening.

I experience the issue when running inside ArcGIS Pro as a python toolbox, as well as running it in VS Code in an interactive terminal or in a Jupyter Notebook.

When my business partners run the tool, the also experience slow run times, and they exclusively use it as part of a Python toolbox. So its not specific to my laptop.

MelindaMorang · ‎09-26-2023

I still can't reproduce the problem using the data and script you sent me via private message. I responded to your private message with more details and further suggestions, so please have a look.

FrankCardone1 · ‎11-10-2023

Melinda, i want to provide an update. After upgrading to ArcGIS Pro 3.2, this issue is no longer happening!