Latest Contributions by GerryGabrisch

This python script selects groups of data on specific days, processes those data, and selects another group of data to process.  The code executes quickly when first executed.  As the code chugs along it gets slower and slower.  There are no nested cursors, no nested loops - just selections and data processing.  Why is code execution getting slower over time?   Any tips or tricks for preventing this type of slowdown is appreciated.   import sys import traceback import os import shutil import arcpy import shapefile try: arcpy.env.overwriteOutput = True #The input shapefile... in_fc = r"C:\GIS\telemetrytags\projected_data\telemetryUTM.shp" #An empty directory to store the geometric average outputs out_dir = r'C:\GIS\telemetrytags\individual_fish_individual_days' #For the first cursor only these fields are needed... fields = ['TagID', 'Date'] #A list to hold all of the tag ids... tag_ids = [] #A list to hold the survey dates ping_dates = [] #Iterate the shapefile and get a list of all unique dates and a list of all unique tags. with arcpy.da.SearchCursor(in_fc, fields) as cursor: for row in cursor: if row[0] not in tag_ids: tag_ids.append(row[0]) if row[1] not in ping_dates: ping_dates.append(row[1]) #Do you need to delete row now with python 3? Doing so did not speed up the code which lags over time... del row #This is the field list for populating the rest of the attribute table of the mean center files... fields = ['Origin','Stock','Sex','FL','Destinatio', 'Mortality_'] #Build the query to select individual tags on individual days... for tag in tag_ids: for ping in ping_dates: #Build the query... query = '\"TagID\" = ' + str(int(tag)) + ' And \"Date\" = timestamp ' +'\'' + str(ping).split(' ')[0] +'\'' print(query) #Select all those tags on that particular day... this_fish, ping_count = arcpy.management.SelectLayerByAttribute(in_fc, selection_type="NEW_SELECTION", where_clause=query) print('ping return count = ', ping_count) #If a fish was not found on a particular day then ignore that selection... if ping_count == 0: pass #If there is only one ping then just rename the original file and move it to the out_dir. if int(ping_count) == 1: out_fc = out_dir + "\\" + str(int(tag)) + '_' + str(ping).split(' ')[0] + '.shp' arcpy.CopyFeatures_management(this_fish, out_fc) #Clean up that table for the final merging of shapefiles... arcpy.DeleteField_management(out_fc, ['MEAS','Pos_ft','CumPos_ft','Pos_RM','CumPos_RM','Time','Channel','Antenna','Power','Latitude','Longitude','NEAR_FID','NEAR_DIST','NEAR_X','NEAR_Y','Data_Inclu']) #More that one ping on a given day needs to be averaged... if int(ping_count) > 1: #Build the output file and path for the averaged data... out_fc = out_dir + "\\" + str(int(tag)) + '_' + str(ping).split(' ')[0] + '.shp' date = str(ping).split(' ')[0] #Create the mean center file based on the geospatial locations... arcpy.stats.MeanCenter(this_fish, out_fc, Weight_Field='Power') #The mean center operation output does not preserve attributes. Create them now... arcpy.AddField_management(out_fc, 'TagID', "SHORT") arcpy.AddField_management(out_fc, 'Date_', "TEXT", field_length='12') arcpy.AddField_management(out_fc, 'Origin', "TEXT", field_length='3') arcpy.AddField_management(out_fc, 'Stock', "TEXT", field_length='20') arcpy.AddField_management(out_fc, 'Sex', "TEXT", field_length='1') arcpy.AddField_management(out_fc, 'FL', "SHORT") arcpy.AddField_management(out_fc, 'Destinatio', "TEXT", field_length='10') arcpy.AddField_management(out_fc, 'Mortality_', "TEXT", field_length='20') #These values are all the same between records, get the values to populate the out_fc... with arcpy.da.SearchCursor(this_fish, fields) as cursor: for row in cursor: Origin = row[0] Stock = row[1] Sex = row[2] FL = int(row[3]) Destinatio = row[4] Mortality_ = row[5] del row arcpy.CalculateField_management(out_fc, 'TagID', int(tag), "PYTHON3") arcpy.CalculateField_management(out_fc, 'Date_', "'"+ date +"'", "PYTHON3") arcpy.CalculateField_management(out_fc, 'Origin', "'"+Origin+"'", "PYTHON3") arcpy.CalculateField_management(out_fc, 'Stock', "'"+Stock+"'", "PYTHON3") arcpy.CalculateField_management(out_fc, 'Sex', "'"+Sex+"'", "PYTHON3") arcpy.CalculateField_management(out_fc, 'FL', FL, "PYTHON3") arcpy.CalculateField_management(out_fc, 'Destinatio', "'"+Destinatio+"'", "PYTHON3") arcpy.CalculateField_management(out_fc, 'Mortality_', "'"+Mortality_+"'", "PYTHON3") except Exception: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info()[1]) msgs = "ArcPy ERRORS:\n" + arcpy.GetMessages(2) + "\n" arcpy.AddError(pymsg) arcpy.AddError(msgs)     ... View more

I am trying to execute a calculate field in arcpy from ArcGIS Pro 2.7.  Here is the relevant code. #a confirmed existing shapefile .... out_fc = r'C:\GIS\telemetrytags\individual_fish_individual_days\112_2019-04-12.shp' # a confirmed and working add field.... arcpy.AddField_management(out_fc, 'TagID', "SHORT") #The existing field name... tagid = "TagID" #The value to populate TagID with...in this case the value of tag is a float (with a value of 112.0) before I #convert it to an int and then string. value = str(int(tag)) #code fails here with the message below. arcpy.CalculateFields_management(out_fc, tagid, value, "PYTHON3") Error Info: Failed to execute. Parameters are not valid. ERROR 000800: The value is not a member of Python 3 | Arcade. ERROR 000728: Field 112 does not exist within table Failed to execute (CalculateFields). 112 is the value I am trying to populate field TagID with and not the field name so I do not understand what is up with that error. Another oddity - when I add my shapefile to ArcGIS pro and check the TagID field it returns that the field is a type Long with is not what I am doing with the calculate field.   Why is this 'the most simple of calculations' failing in arcpy for ArcGIS Pro 2.7     ... View more